58 changes: 29 additions & 29 deletions bolt/include/bolt/Passes/BinaryPasses.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ class BinaryFunctionPass {
/// this pass is completed (printPass() must have returned true).
virtual bool shouldPrint(const BinaryFunction &BF) const;

/// Execute this pass on the given functions.
virtual void runOnFunctions(BinaryContext &BC) = 0;
virtual Error runOnFunctions(BinaryContext &BC) = 0;
};

/// A pass to print program-wide dynostats.
Expand All @@ -70,18 +69,19 @@ class DynoStatsPrintPass : public BinaryFunctionPass {

bool shouldPrint(const BinaryFunction &BF) const override { return false; }

void runOnFunctions(BinaryContext &BC) override {
Error runOnFunctions(BinaryContext &BC) override {
const DynoStats NewDynoStats =
getDynoStats(BC.getBinaryFunctions(), BC.isAArch64());
const bool Changed = (NewDynoStats != PrevDynoStats);
outs() << "BOLT-INFO: program-wide dynostats " << Title
<< (Changed ? "" : " (no change)") << ":\n\n"
<< PrevDynoStats;
BC.outs() << "BOLT-INFO: program-wide dynostats " << Title
<< (Changed ? "" : " (no change)") << ":\n\n"
<< PrevDynoStats;
if (Changed) {
outs() << '\n';
NewDynoStats.print(outs(), &PrevDynoStats, BC.InstPrinter.get());
BC.outs() << '\n';
NewDynoStats.print(BC.outs(), &PrevDynoStats, BC.InstPrinter.get());
}
outs() << '\n';
BC.outs() << '\n';
return Error::success();
}
};

Expand All @@ -100,7 +100,7 @@ class NormalizeCFG : public BinaryFunctionPass {

const char *getName() const override { return "normalize CFG"; }

void runOnFunctions(BinaryContext &) override;
Error runOnFunctions(BinaryContext &) override;
};

/// Detect and eliminate unreachable basic blocks. We could have those
Expand All @@ -119,7 +119,7 @@ class EliminateUnreachableBlocks : public BinaryFunctionPass {
bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
}
void runOnFunctions(BinaryContext &) override;
Error runOnFunctions(BinaryContext &) override;
};

// Reorder the basic blocks for each function based on hotness.
Expand Down Expand Up @@ -165,7 +165,7 @@ class ReorderBasicBlocks : public BinaryFunctionPass {

const char *getName() const override { return "reorder-blocks"; }
bool shouldPrint(const BinaryFunction &BF) const override;
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Sync local branches with CFG.
Expand All @@ -175,7 +175,7 @@ class FixupBranches : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "fix-branches"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Fix the CFI state and exception handling information after all other
Expand All @@ -186,7 +186,7 @@ class FinalizeFunctions : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "finalize-functions"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Perform any necessary adjustments for functions that do not fit into their
Expand All @@ -198,7 +198,7 @@ class CheckLargeFunctions : public BinaryFunctionPass {

const char *getName() const override { return "check-large-functions"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

bool shouldOptimize(const BinaryFunction &BF) const override;
};
Expand All @@ -210,7 +210,7 @@ class LowerAnnotations : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "lower-annotations"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Clean the state of the MC representation before sending it to emission
Expand All @@ -220,7 +220,7 @@ class CleanMCState : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "clean-mc-state"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// An optimization to simplify conditional tail calls by removing
Expand Down Expand Up @@ -292,7 +292,7 @@ class SimplifyConditionalTailCalls : public BinaryFunctionPass {
bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
}
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Convert instructions to the form with the minimum operand width.
Expand All @@ -305,7 +305,7 @@ class ShortenInstructions : public BinaryFunctionPass {

const char *getName() const override { return "shorten-instructions"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Perform simple peephole optimizations.
Expand Down Expand Up @@ -339,7 +339,7 @@ class Peepholes : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "peepholes"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// An optimization to simplify loads from read-only sections.The pass converts
Expand Down Expand Up @@ -370,7 +370,7 @@ class SimplifyRODataLoads : public BinaryFunctionPass {
bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
}
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Assign output sections to all functions.
Expand All @@ -379,7 +379,7 @@ class AssignSections : public BinaryFunctionPass {
explicit AssignSections() : BinaryFunctionPass(false) {}

const char *getName() const override { return "assign-sections"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Compute and report to the user the imbalance in flow equations for all
Expand All @@ -394,7 +394,7 @@ class PrintProfileStats : public BinaryFunctionPass {

const char *getName() const override { return "profile-stats"; }
bool shouldPrint(const BinaryFunction &) const override { return false; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Prints a list of the top 100 functions sorted by a set of
Expand All @@ -406,7 +406,7 @@ class PrintProgramStats : public BinaryFunctionPass {

const char *getName() const override { return "print-stats"; }
bool shouldPrint(const BinaryFunction &) const override { return false; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Pass for lowering any instructions that we have raised and that have
Expand All @@ -418,7 +418,7 @@ class InstructionLowering : public BinaryFunctionPass {

const char *getName() const override { return "inst-lowering"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Pass for stripping 'repz' from 'repz retq' sequence of instructions.
Expand All @@ -429,7 +429,7 @@ class StripRepRet : public BinaryFunctionPass {

const char *getName() const override { return "strip-rep-ret"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Pass for inlining calls to memcpy using 'rep movsb' on X86.
Expand All @@ -440,7 +440,7 @@ class InlineMemcpy : public BinaryFunctionPass {

const char *getName() const override { return "inline-memcpy"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Pass for specializing memcpy for a size of 1 byte.
Expand All @@ -461,7 +461,7 @@ class SpecializeMemcpy1 : public BinaryFunctionPass {

const char *getName() const override { return "specialize-memcpy"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

/// Pass to remove nops in code
Expand All @@ -475,7 +475,7 @@ class RemoveNops : public BinaryFunctionPass {
const char *getName() const override { return "remove-nops"; }

/// Pass entry point
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

enum FrameOptimizationType : char {
Expand Down
4 changes: 2 additions & 2 deletions bolt/include/bolt/Passes/CMOVConversion.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class CMOVConversion : public BinaryFunctionPass {
}
double getMPRatio() { return (double)RemovedMP / PossibleMP; }

void dump();
void dumpTo(raw_ostream &OS);
};
// BinaryContext-wide stats
Stats Global;
Expand All @@ -76,7 +76,7 @@ class CMOVConversion : public BinaryFunctionPass {

const char *getName() const override { return "CMOV conversion"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
6 changes: 5 additions & 1 deletion bolt/include/bolt/Passes/CacheMetrics.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@
#include <vector>

namespace llvm {

class raw_ostream;

namespace bolt {
class BinaryFunction;
namespace CacheMetrics {

/// Calculate and print various metrics related to instruction cache performance
void printAll(const std::vector<BinaryFunction *> &BinaryFunctions);
void printAll(raw_ostream &OS,
const std::vector<BinaryFunction *> &BinaryFunctions);

} // namespace CacheMetrics
} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/FixRISCVCallsPass.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class FixRISCVCallsPass : public BinaryFunctionPass {
const char *getName() const override { return "fix-riscv-calls"; }

/// Pass entry point
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/FixRelaxationPass.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class FixRelaxations : public BinaryFunctionPass {
const char *getName() const override { return "fix-relaxations"; }

/// Pass entry point
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
6 changes: 3 additions & 3 deletions bolt/include/bolt/Passes/FrameOptimizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ class FrameOptimizerPass : public BinaryFunctionPass {
void removeUnusedStores(const FrameAnalysis &FA, BinaryFunction &BF);

/// Perform shrinkwrapping step
void performShrinkWrapping(const RegAnalysis &RA, const FrameAnalysis &FA,
BinaryContext &BC);
Error performShrinkWrapping(const RegAnalysis &RA, const FrameAnalysis &FA,
BinaryContext &BC);

public:
explicit FrameOptimizerPass(const cl::opt<bool> &PrintPass)
Expand All @@ -108,7 +108,7 @@ class FrameOptimizerPass : public BinaryFunctionPass {
const char *getName() const override { return "frame-optimizer"; }

/// Pass entry point
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF) && FuncsChanged.count(&BF) > 0;
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/Hugify.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class HugePage : public BinaryFunctionPass {
public:
HugePage(const cl::opt<bool> &PrintPass) : BinaryFunctionPass(PrintPass) {}

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

const char *getName() const override { return "HugePage"; }
};
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/IdenticalCodeFolding.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class IdenticalCodeFolding : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "identical-code-folding"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/IndirectCallPromotion.h
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ class IndirectCallPromotion : public BinaryFunctionPass {
return BF.isSimple() && !BF.isIgnored() && BF.hasProfile() &&
!BF.hasUnknownControlFlow();
}
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/Inliner.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ class Inliner : public BinaryFunctionPass {
return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
}

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/Instrumentation.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class Instrumentation : public BinaryFunctionPass {
Summary(std::make_unique<InstrumentationSummary>()) {}

/// Modifies all functions by inserting instrumentation code (first step)
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

const char *getName() const override { return "instrumentation"; }

Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/JTFootprintReduction.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class JTFootprintReduction : public BinaryFunctionPass {
bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
}
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
6 changes: 3 additions & 3 deletions bolt/include/bolt/Passes/LongJmp.h
Original file line number Diff line number Diff line change
Expand Up @@ -131,14 +131,14 @@ class LongJmpPass : public BinaryFunctionPass {
uint64_t DotAddress) const;

/// Expand the range of the stub in StubBB if necessary
bool relaxStub(BinaryBasicBlock &StubBB);
Error relaxStub(BinaryBasicBlock &StubBB, bool &Modified);

/// Helper to resolve a symbol address according to our tentative layout
uint64_t getSymbolAddress(const BinaryContext &BC, const MCSymbol *Target,
const BinaryBasicBlock *TgtBB) const;

/// Relax function by adding necessary stubs or relaxing existing stubs
bool relax(BinaryFunction &BF);
Error relax(BinaryFunction &BF, bool &Modified);

public:
/// BinaryPass public interface
Expand All @@ -148,7 +148,7 @@ class LongJmpPass : public BinaryFunctionPass {

const char *getName() const override { return "long-jmp"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};
} // namespace bolt
} // namespace llvm
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/LoopInversionPass.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class LoopInversionPass : public BinaryFunctionPass {
const char *getName() const override { return "loop-inversion-opt"; }

/// Pass entry point
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
bool runOnFunction(BinaryFunction &Function);
};

Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/PLTCall.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class PLTCall : public BinaryFunctionPass {
bool shouldPrint(const BinaryFunction &BF) const override {
return BinaryFunctionPass::shouldPrint(BF);
}
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/PatchEntries.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class PatchEntries : public BinaryFunctionPass {
explicit PatchEntries() : BinaryFunctionPass(false) {}

const char *getName() const override { return "patch-entries"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/RegReAssign.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class RegReAssign : public BinaryFunctionPass {
return BinaryFunctionPass::shouldPrint(BF) && FuncsChanged.count(&BF) > 0;
}

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};
} // namespace bolt
} // namespace llvm
Expand Down
5 changes: 3 additions & 2 deletions bolt/include/bolt/Passes/ReorderData.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ class ReorderData : public BinaryFunctionPass {
sortedByFunc(BinaryContext &BC, const BinarySection &Section,
std::map<uint64_t, BinaryFunction> &BFs) const;

void printOrder(const BinarySection &Section, DataOrder::const_iterator Begin,
void printOrder(BinaryContext &BC, const BinarySection &Section,
DataOrder::const_iterator Begin,
DataOrder::const_iterator End) const;

/// Set the ordering of the section with \p SectionName. \p NewOrder is a
Expand All @@ -51,7 +52,7 @@ class ReorderData : public BinaryFunctionPass {

const char *getName() const override { return "reorder-data"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
8 changes: 4 additions & 4 deletions bolt/include/bolt/Passes/ReorderFunctions.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ class Cluster;
class ReorderFunctions : public BinaryFunctionPass {
BinaryFunctionCallGraph Cg;

void reorder(std::vector<Cluster> &&Clusters,
void reorder(BinaryContext &BC, std::vector<Cluster> &&Clusters,
std::map<uint64_t, BinaryFunction> &BFs);

void printStats(const std::vector<Cluster> &Clusters,
void printStats(BinaryContext &BC, const std::vector<Cluster> &Clusters,
const std::vector<uint64_t> &FuncAddr);

public:
Expand All @@ -42,9 +42,9 @@ class ReorderFunctions : public BinaryFunctionPass {
: BinaryFunctionPass(PrintPass) {}

const char *getName() const override { return "reorder-functions"; }
void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

static std::vector<std::string> readFunctionOrderFile();
static Error readFunctionOrderFile(std::vector<std::string> &FunctionNames);
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/RetpolineInsertion.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ class RetpolineInsertion : public BinaryFunctionPass {

const char *getName() const override { return "retpoline-insertion"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
26 changes: 14 additions & 12 deletions bolt/include/bolt/Passes/ShrinkWrapping.h
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,9 @@ class ShrinkWrapping {
/// If \p CreatePushOrPop is true, create a push/pop instead. Current SP/FP
/// values, as determined by StackPointerTracking, should be informed via
/// \p SPVal and \p FPVal in order to emit the correct offset form SP/FP.
MCInst createStackAccess(int SPVal, int FPVal, const FrameIndexEntry &FIE,
bool CreatePushOrPop);
Expected<MCInst> createStackAccess(int SPVal, int FPVal,
const FrameIndexEntry &FIE,
bool CreatePushOrPop);

/// Update the CFI referenced by \p Inst with \p NewOffset, if the CFI has
/// an offset.
Expand All @@ -484,22 +485,23 @@ class ShrinkWrapping {
/// InsertionPoint for other instructions that need to be inserted at the same
/// original location, since this insertion may have invalidated the previous
/// location.
BBIterTy processInsertion(BBIterTy InsertionPoint, BinaryBasicBlock *CurBB,
const WorklistItem &Item, int64_t SPVal,
int64_t FPVal);
Expected<BBIterTy> processInsertion(BBIterTy InsertionPoint,
BinaryBasicBlock *CurBB,
const WorklistItem &Item, int64_t SPVal,
int64_t FPVal);

/// Auxiliary function to processInsertions(), helping perform all the
/// insertion tasks in the todo list associated with a single insertion point.
/// Return true if at least one insertion was performed.
BBIterTy processInsertionsList(BBIterTy InsertionPoint,
BinaryBasicBlock *CurBB,
std::vector<WorklistItem> &TodoList,
int64_t SPVal, int64_t FPVal);
Expected<BBIterTy> processInsertionsList(BBIterTy InsertionPoint,
BinaryBasicBlock *CurBB,
std::vector<WorklistItem> &TodoList,
int64_t SPVal, int64_t FPVal);

/// Apply all insertion todo tasks regarding insertion of new stores/loads or
/// push/pops at annotated points. Return false if the entire function had
/// no todo tasks annotation and this pass has nothing to do.
bool processInsertions();
Expected<bool> processInsertions();

/// Apply all deletion todo tasks (or tasks to change a push/pop to a memory
/// access no-op)
Expand All @@ -519,9 +521,9 @@ class ShrinkWrapping {
BC.MIB->removeAnnotation(Inst, getAnnotationIndex());
}

bool perform(bool HotOnly = false);
Expected<bool> perform(bool HotOnly = false);

static void printStats();
static void printStats(BinaryContext &BC);
};

} // end namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/SplitFunctions.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ class SplitFunctions : public BinaryFunctionPass {

const char *getName() const override { return "split-functions"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/StokeInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ class StokeInfo : public BinaryFunctionPass {
bool checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,
RegAnalysis &RA, StokeFuncInfo &FuncInfo);

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/TailDuplication.h
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ class TailDuplication : public BinaryFunctionPass {

const char *getName() const override { return "tail duplication"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/ThreeWayBranch.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class ThreeWayBranch : public BinaryFunctionPass {

const char *getName() const override { return "three way branch"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/ValidateInternalCalls.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class ValidateInternalCalls : public BinaryFunctionPass {

const char *getName() const override { return "validate-internal-calls"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

private:
/// Fix the CFG to take into consideration internal calls that do not
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/ValidateMemRefs.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class ValidateMemRefs : public BinaryFunctionPass {

const char *getName() const override { return "validate-mem-refs"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;

private:
bool checkAndFixJTReference(BinaryFunction &BF, MCInst &Inst,
Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Passes/VeneerElimination.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class VeneerElimination : public BinaryFunctionPass {

const char *getName() const override { return "veneer-elimination"; }

void runOnFunctions(BinaryContext &BC) override;
Error runOnFunctions(BinaryContext &BC) override;
};

} // namespace bolt
Expand Down
9 changes: 8 additions & 1 deletion bolt/include/bolt/Profile/BoltAddressTranslation.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <map>
#include <optional>
#include <system_error>
#include <unordered_map>

namespace llvm {
class raw_ostream;
Expand Down Expand Up @@ -85,7 +86,7 @@ class BoltAddressTranslation {

/// Read the serialized address translation tables and load them internally
/// in memory. Return a parse error if failed.
std::error_code parse(StringRef Buf);
std::error_code parse(raw_ostream &OS, StringRef Buf);

/// Dump the parsed address translation tables
void dump(raw_ostream &OS);
Expand All @@ -111,6 +112,9 @@ class BoltAddressTranslation {
/// addresses when aggregating profile
bool enabledFor(llvm::object::ELFObjectFileBase *InputFile) const;

/// Save function and basic block hashes used for metadata dump.
void saveMetadata(BinaryContext &BC);

private:
/// Helper to update \p Map by inserting one or more BAT entries reflecting
/// \p BB for function located at \p FuncAddress. At least one entry will be
Expand Down Expand Up @@ -140,6 +144,9 @@ class BoltAddressTranslation {

std::map<uint64_t, MapTy> Maps;

using BBHashMap = std::unordered_map<uint32_t, size_t>;
std::unordered_map<uint64_t, std::pair<size_t, BBHashMap>> FuncHashes;

/// Links outlined cold bocks to their original function
std::map<uint64_t, uint64_t> ColdPartSource;

Expand Down
4 changes: 2 additions & 2 deletions bolt/include/bolt/Rewrite/BinaryPassManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ class BinaryFunctionPassManager {
}

/// Run all registered passes in the order they were added.
void runPasses();
Error runPasses();

/// Runs all enabled implemented passes on all functions.
static void runAllPasses(BinaryContext &BC);
static Error runAllPasses(BinaryContext &BC);
};

} // namespace bolt
Expand Down
7 changes: 5 additions & 2 deletions bolt/include/bolt/Rewrite/RewriteInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,14 @@ class RewriteInstance {
// construction. Constructors can’t return errors, so clients must test \p Err
// after the object is constructed. Use `create` method instead.
RewriteInstance(llvm::object::ELFObjectFileBase *File, const int Argc,
const char *const *Argv, StringRef ToolPath, Error &Err);
const char *const *Argv, StringRef ToolPath,
raw_ostream &Stdout, raw_ostream &Stderr, Error &Err);

static Expected<std::unique_ptr<RewriteInstance>>
create(llvm::object::ELFObjectFileBase *File, const int Argc,
const char *const *Argv, StringRef ToolPath);
const char *const *Argv, StringRef ToolPath,
raw_ostream &Stdout = llvm::outs(),
raw_ostream &Stderr = llvm::errs());
~RewriteInstance();

/// Assign profile from \p Filename to this instance.
Expand Down
36 changes: 18 additions & 18 deletions bolt/lib/Core/BinaryBasicBlock.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
// Work on the assumption that jump table blocks don't
// have a conditional successor.
Valid = false;
errs() << "BOLT-WARNING: Jump table successor " << Succ->getName()
<< " not contained in the jump table.\n";
BC.errs() << "BOLT-WARNING: Jump table successor " << Succ->getName()
<< " not contained in the jump table.\n";
}
}
// If there are any leftover entries in the jump table, they
Expand All @@ -103,8 +103,8 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
Valid &= (Sym == Function->getFunctionEndLabel() ||
Sym == Function->getFunctionEndLabel(getFragmentNum()));
if (!Valid) {
errs() << "BOLT-WARNING: Jump table contains illegal entry: "
<< Sym->getName() << "\n";
BC.errs() << "BOLT-WARNING: Jump table contains illegal entry: "
<< Sym->getName() << "\n";
}
}
}
Expand Down Expand Up @@ -141,11 +141,11 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
}
}
if (!Valid) {
errs() << "BOLT-WARNING: CFG invalid in " << *getFunction() << " @ "
<< getName() << "\n";
BC.errs() << "BOLT-WARNING: CFG invalid in " << *getFunction() << " @ "
<< getName() << "\n";
if (JT) {
errs() << "Jump Table instruction addr = 0x"
<< Twine::utohexstr(BC.MIB->getJumpTable(*Inst)) << "\n";
BC.errs() << "Jump Table instruction addr = 0x"
<< Twine::utohexstr(BC.MIB->getJumpTable(*Inst)) << "\n";
JT->print(errs());
}
getFunction()->dump();
Expand Down Expand Up @@ -520,9 +520,9 @@ uint32_t BinaryBasicBlock::getNumPseudos() const {
++N;

if (N != NumPseudos) {
errs() << "BOLT-ERROR: instructions for basic block " << getName()
<< " in function " << *Function << ": calculated pseudos " << N
<< ", set pseudos " << NumPseudos << ", size " << size() << '\n';
BC.errs() << "BOLT-ERROR: instructions for basic block " << getName()
<< " in function " << *Function << ": calculated pseudos " << N
<< ", set pseudos " << NumPseudos << ", size " << size() << '\n';
llvm_unreachable("pseudos mismatch");
}
#endif
Expand Down Expand Up @@ -559,18 +559,18 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const {
void BinaryBasicBlock::dump() const {
BinaryContext &BC = Function->getBinaryContext();
if (Label)
outs() << Label->getName() << ":\n";
BC.printInstructions(outs(), Instructions.begin(), Instructions.end(),
BC.outs() << Label->getName() << ":\n";
BC.printInstructions(BC.outs(), Instructions.begin(), Instructions.end(),
getOffset(), Function);
outs() << "preds:";
BC.outs() << "preds:";
for (auto itr = pred_begin(); itr != pred_end(); ++itr) {
outs() << " " << (*itr)->getName();
BC.outs() << " " << (*itr)->getName();
}
outs() << "\nsuccs:";
BC.outs() << "\nsuccs:";
for (auto itr = succ_begin(); itr != succ_end(); ++itr) {
outs() << " " << (*itr)->getName();
BC.outs() << " " << (*itr)->getName();
}
outs() << "\n";
BC.outs() << "\n";
}

uint64_t BinaryBasicBlock::estimateSize(const MCCodeEmitter *Emitter) const {
Expand Down
220 changes: 135 additions & 85 deletions bolt/lib/Core/BinaryContext.cpp

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions bolt/lib/Core/BinaryEmitter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,8 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());

if (opts::Verbosity && !OnBehalfOf)
outs() << "BOLT-INFO: emitting constant island for function " << BF << "\n";
BC.outs() << "BOLT-INFO: emitting constant island for function " << BF
<< "\n";

// We split the island into smaller blocks and output labels between them.
auto IS = Islands.Offsets.begin();
Expand Down Expand Up @@ -766,15 +767,15 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
return;

if (opts::PrintJumpTables)
outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";
BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";

for (auto &JTI : BF.jumpTables()) {
JumpTable &JT = *JTI.second;
// Only emit shared jump tables once, when processing the first parent
if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
continue;
if (opts::PrintJumpTables)
JT.print(outs());
JT.print(BC.outs());
if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
JT.updateOriginal();
} else {
Expand Down
236 changes: 130 additions & 106 deletions bolt/lib/Core/BinaryFunction.cpp

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion bolt/lib/Core/BinarySection.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ BinarySection::~BinarySection() {

if (!isAllocatable() && !hasValidSectionID() &&
(!hasSectionRef() ||
OutputContents.data() != getContents(Section).data())) {
OutputContents.data() != getContentsOrQuit(Section).data())) {
delete[] getOutputData();
}
}
Expand Down
68 changes: 36 additions & 32 deletions bolt/lib/Core/DIEBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ uint32_t DIEBuilder::allocDIE(const DWARFUnit &DU, const DWARFDie &DDie,
void DIEBuilder::constructFromUnit(DWARFUnit &DU) {
std::optional<uint32_t> UnitId = getUnitId(DU);
if (!UnitId) {
errs() << "BOLT-WARNING: [internal-dwarf-error]: "
<< "Skip Unit at " << Twine::utohexstr(DU.getOffset()) << "\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: "
<< "Skip Unit at " << Twine::utohexstr(DU.getOffset()) << "\n";
return;
}

Expand Down Expand Up @@ -178,8 +178,9 @@ void DIEBuilder::constructFromUnit(DWARFUnit &DU) {
getState().CloneUnitCtxMap[*UnitId].IsConstructed = true;
}

DIEBuilder::DIEBuilder(DWARFContext *DwarfContext, bool IsDWO)
: DwarfContext(DwarfContext), IsDWO(IsDWO) {}
DIEBuilder::DIEBuilder(BinaryContext &BC, DWARFContext *DwarfContext,
bool IsDWO)
: BC(BC), DwarfContext(DwarfContext), IsDWO(IsDWO) {}

static unsigned int getCUNum(DWARFContext *DwarfContext, bool IsDWO) {
unsigned int CUNum = IsDWO ? DwarfContext->getNumDWOCompileUnits()
Expand Down Expand Up @@ -475,19 +476,21 @@ DWARFDie DIEBuilder::resolveDIEReference(
allocDIE(*RefCU, RefDie, getState().DIEAlloc, *UnitId);
return RefDie;
}
errs() << "BOLT-WARNING: [internal-dwarf-error]: invalid referenced DIE "
"at offset: "
<< Twine::utohexstr(RefOffset) << ".\n";
BC.errs()
<< "BOLT-WARNING: [internal-dwarf-error]: invalid referenced DIE "
"at offset: "
<< Twine::utohexstr(RefOffset) << ".\n";

} else {
errs() << "BOLT-WARNING: [internal-dwarf-error]: could not parse "
"referenced DIE at offset: "
<< Twine::utohexstr(RefOffset) << ".\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: could not parse "
"referenced DIE at offset: "
<< Twine::utohexstr(RefOffset) << ".\n";
}
} else {
errs() << "BOLT-WARNING: [internal-dwarf-error]: could not find referenced "
"CU. Referenced DIE offset: "
<< Twine::utohexstr(RefOffset) << ".\n";
BC.errs()
<< "BOLT-WARNING: [internal-dwarf-error]: could not find referenced "
"CU. Referenced DIE offset: "
<< Twine::utohexstr(RefOffset) << ".\n";
}
return DWARFDie();
}
Expand Down Expand Up @@ -516,8 +519,8 @@ void DIEBuilder::cloneDieReferenceAttribute(
if (!DieInfo.Die) {
assert(Ref > InputDIE.getOffset());
(void)Ref;
errs() << "BOLT-WARNING: [internal-dwarf-error]: encounter unexpected "
"unallocated DIE. Should be alloc!\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: encounter unexpected "
"unallocated DIE. Should be alloc!\n";
// We haven't cloned this DIE yet. Just create an empty one and
// store it. It'll get really cloned when we process it.
DieInfo.Die = DIE::get(getState().DIEAlloc, dwarf::Tag(RefDie.getTag()));
Expand Down Expand Up @@ -580,8 +583,8 @@ bool DIEBuilder::cloneExpression(const DataExtractor &Data,
(Description.Op.size() == 2 &&
Description.Op[1] == Encoding::BaseTypeRef &&
Description.Op[0] != Encoding::Size1))
outs() << "BOLT-WARNING: [internal-dwarf-error]: unsupported DW_OP "
"encoding.\n";
BC.outs() << "BOLT-WARNING: [internal-dwarf-error]: unsupported DW_OP "
"encoding.\n";

if ((Description.Op.size() == 1 &&
Description.Op[0] == Encoding::BaseTypeRef) ||
Expand Down Expand Up @@ -616,9 +619,9 @@ bool DIEBuilder::cloneExpression(const DataExtractor &Data,
Offset = Stage == CloneExpressionStage::INIT ? RefOffset
: Clone->getOffset();
else
errs() << "BOLT-WARNING: [internal-dwarf-error]: base type ref "
"doesn't point to "
"DW_TAG_base_type.\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: base type ref "
"doesn't point to "
"DW_TAG_base_type.\n";
}
}
uint8_t ULEB[16];
Expand Down Expand Up @@ -652,8 +655,9 @@ void DIEBuilder::cloneBlockAttribute(
U.getVersion())) {
Block = new (getState().DIEAlloc) DIEBlock;
} else {
errs() << "BOLT-WARNING: [internal-dwarf-error]: Unexpected Form value in "
"cloneBlockAttribute\n";
BC.errs()
<< "BOLT-WARNING: [internal-dwarf-error]: Unexpected Form value in "
"cloneBlockAttribute\n";
return;
}
Attr = Loc ? static_cast<DIEValueList *>(Loc)
Expand Down Expand Up @@ -720,9 +724,9 @@ void DIEBuilder::cloneScalarAttribute(
else if (auto OptionalValue = Val.getAsSectionOffset())
Value = *OptionalValue;
else {
errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported scalar "
"attribute form. Dropping "
"attribute.\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported scalar "
"attribute form. Dropping "
"attribute.\n";
return;
}

Expand All @@ -743,9 +747,9 @@ void DIEBuilder::cloneLoclistAttrubute(
else if (auto OptionalValue = Val.getAsSectionOffset())
Value = OptionalValue;
else
errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported scalar "
"attribute form. Dropping "
"attribute.\n";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported scalar "
"attribute form. Dropping "
"attribute.\n";

if (!Value.has_value())
return;
Expand Down Expand Up @@ -808,10 +812,10 @@ void DIEBuilder::cloneAttribute(
cloneRefsigAttribute(Die, AttrSpec, Val);
break;
default:
errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported attribute "
"form " +
dwarf::FormEncodingString(AttrSpec.Form).str() +
" in cloneAttribute. Dropping.";
BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: Unsupported attribute "
"form " +
dwarf::FormEncodingString(AttrSpec.Form).str() +
" in cloneAttribute. Dropping.";
}
}
void DIEBuilder::assignAbbrev(DIEAbbrev &Abbrev) {
Expand Down
47 changes: 20 additions & 27 deletions bolt/lib/Core/DebugData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ template <typename DebugVector, typename ListEntry, typename DebugAddressEntry>
static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
DebugAddrWriter &AddrWriter, DWARFUnit &CU,
uint32_t &Index, const ListEntry BaseAddressx,
const ListEntry OffsetPair, const ListEntry EndOfList,
const ListEntry OffsetPair,
const std::function<void(uint32_t)> &Func) {
if (Entries.size() < 2)
return false;
Expand All @@ -241,7 +241,9 @@ static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
const DebugAddressEntry &Entry = Entries[Index];
if (Entry.LowPC == 0)
break;
assert(Base <= Entry.LowPC && "Entry base is higher than low PC");
// In case rnglists or loclists are not sorted.
if (Base > Entry.LowPC)
break;
uint32_t StartOffset = Entry.LowPC - Base;
uint32_t EndOffset = Entry.HighPC - Base;
if (encodeULEB128(EndOffset, TempBuffer) > 2)
Expand All @@ -266,8 +268,6 @@ static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
encodeULEB128(OffsetEntry.EndOffset, OS);
Func(OffsetEntry.Index);
}
support::endian::write(OS, static_cast<uint8_t>(EndOfList),
llvm::endianness::little);
return true;
}

Expand All @@ -276,19 +276,17 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
std::lock_guard<std::mutex> Lock(WriterMutex);

RangeEntries.push_back(CurrentOffset);
bool WrittenStartxLength = false;
std::sort(
Ranges.begin(), Ranges.end(),
[](const DebugAddressRange &R1, const DebugAddressRange &R2) -> bool {
return R1.LowPC < R2.LowPC;
});
for (unsigned I = 0; I < Ranges.size();) {
WrittenStartxLength = false;
if (emitWithBase<DebugAddressRangesVector, dwarf::RnglistEntries,
DebugAddressRange>(
*CUBodyStream, Ranges, *AddrWriter, *CU, I,
dwarf::DW_RLE_base_addressx, dwarf::DW_RLE_offset_pair,
dwarf::DW_RLE_end_of_list, [](uint32_t Index) -> void {}))
DebugAddressRange>(*CUBodyStream, Ranges, *AddrWriter, *CU,
I, dwarf::DW_RLE_base_addressx,
dwarf::DW_RLE_offset_pair,
[](uint32_t Index) -> void {}))
continue;

const DebugAddressRange &Range = Ranges[I];
Expand All @@ -299,12 +297,11 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
encodeULEB128(Index, *CUBodyStream);
encodeULEB128(Range.HighPC - Range.LowPC, *CUBodyStream);
++I;
WrittenStartxLength = true;
}
if (WrittenStartxLength)
support::endian::write(*CUBodyStream,
static_cast<uint8_t>(dwarf::DW_RLE_end_of_list),
llvm::endianness::little);

support::endian::write(*CUBodyStream,
static_cast<uint8_t>(dwarf::DW_RLE_end_of_list),
llvm::endianness::little);
CurrentOffset = CUBodyBuffer->size();
return RangeEntries.size() - 1;
}
Expand Down Expand Up @@ -688,20 +685,18 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
}

std::vector<uint64_t> OffsetsArray;
bool WrittenStartxLength = false;
auto writeExpression = [&](uint32_t Index) -> void {
const DebugLocationEntry &Entry = LocList[Index];
encodeULEB128(Entry.Expr.size(), LocBodyStream);
LocBodyStream << StringRef(
reinterpret_cast<const char *>(Entry.Expr.data()), Entry.Expr.size());
};
for (unsigned I = 0; I < LocList.size();) {
WrittenStartxLength = false;
if (emitWithBase<DebugLocationsVector, dwarf::LoclistEntries,
DebugLocationEntry>(
LocBodyStream, LocList, AddrWriter, CU, I,
dwarf::DW_LLE_base_addressx, dwarf::DW_LLE_offset_pair,
dwarf::DW_LLE_end_of_list, writeExpression))
DebugLocationEntry>(LocBodyStream, LocList, AddrWriter, CU,
I, dwarf::DW_LLE_base_addressx,
dwarf::DW_LLE_offset_pair,
writeExpression))
continue;

const DebugLocationEntry &Entry = LocList[I];
Expand All @@ -713,13 +708,11 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
encodeULEB128(Entry.HighPC - Entry.LowPC, LocBodyStream);
writeExpression(I);
++I;
WrittenStartxLength = true;
}

if (WrittenStartxLength)
support::endian::write(LocBodyStream,
static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
llvm::endianness::little);
support::endian::write(LocBodyStream,
static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
llvm::endianness::little);
}

void DebugLoclistWriter::addList(DIEBuilder &DIEBldr, DIE &Die,
Expand Down Expand Up @@ -1185,7 +1178,7 @@ static void parseAndPopulateDebugLineStr(BinarySection &LineStrSection,
Error Err = Error::success();
const char *CStr = StrData.getCStr(&Offset, &Err);
if (Err) {
errs() << "BOLT-ERROR: could not extract string from .debug_line_str";
BC.errs() << "BOLT-ERROR: could not extract string from .debug_line_str";
continue;
}
const size_t NewOffset = LineStr.addString(CStr);
Expand Down
2 changes: 1 addition & 1 deletion bolt/lib/Core/DynoStats.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other,
printStatWithDelta(Desc[Stat], Stats[Stat], Other ? (*Other)[Stat] : 0);
}
if (opts::PrintDynoOpcodeStat && Printer) {
outs() << "\nProgram-wide opcode histogram:\n";
OS << "\nProgram-wide opcode histogram:\n";
OS << " Opcode, Execution Count, Max Exec Count, "
"Function Name:Offset ...\n";
std::vector<std::pair<uint64_t, unsigned>> SortedHistogram;
Expand Down
104 changes: 55 additions & 49 deletions bolt/lib/Core/Exceptions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,12 @@ namespace bolt {
// site table will be the same size as GCC uses uleb encodings for PC offsets.
//
// Note: some functions have LSDA entries with 0 call site entries.
void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uint64_t LSDASectionAddress) {
Error BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uint64_t LSDASectionAddress) {
assert(CurrentState == State::Disassembled && "unexpected function state");

if (!getLSDAAddress())
return;
return Error::success();

DWARFDataExtractor Data(
StringRef(reinterpret_cast<const char *>(LSDASectionData.data()),
Expand All @@ -119,9 +119,9 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
std::optional<uint64_t> MaybeLPStart = Data.getEncodedPointer(
&Offset, LPStartEncoding, Offset + LSDASectionAddress);
if (!MaybeLPStart) {
errs() << "BOLT-ERROR: unsupported LPStartEncoding: "
<< (unsigned)LPStartEncoding << '\n';
exit(1);
BC.errs() << "BOLT-ERROR: unsupported LPStartEncoding: "
<< (unsigned)LPStartEncoding << '\n';
return createFatalBOLTError("");
}
LPStart = *MaybeLPStart;
}
Expand All @@ -136,13 +136,14 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
}

if (opts::PrintExceptions) {
outs() << "[LSDA at 0x" << Twine::utohexstr(getLSDAAddress())
<< " for function " << *this << "]:\n";
outs() << "LPStart Encoding = 0x" << Twine::utohexstr(LPStartEncoding)
<< '\n';
outs() << "LPStart = 0x" << Twine::utohexstr(LPStart) << '\n';
outs() << "TType Encoding = 0x" << Twine::utohexstr(TTypeEncoding) << '\n';
outs() << "TType End = " << TTypeEnd << '\n';
BC.outs() << "[LSDA at 0x" << Twine::utohexstr(getLSDAAddress())
<< " for function " << *this << "]:\n";
BC.outs() << "LPStart Encoding = 0x" << Twine::utohexstr(LPStartEncoding)
<< '\n';
BC.outs() << "LPStart = 0x" << Twine::utohexstr(LPStart) << '\n';
BC.outs() << "TType Encoding = 0x" << Twine::utohexstr(TTypeEncoding)
<< '\n';
BC.outs() << "TType End = " << TTypeEnd << '\n';
}

// Table to store list of indices in type table. Entries are uleb128 values.
Expand All @@ -166,9 +167,9 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uint64_t ActionTableStart = CallSiteTableEnd;

if (opts::PrintExceptions) {
outs() << "CallSite Encoding = " << (unsigned)CallSiteEncoding << '\n';
outs() << "CallSite table length = " << CallSiteTableLength << '\n';
outs() << '\n';
BC.outs() << "CallSite Encoding = " << (unsigned)CallSiteEncoding << '\n';
BC.outs() << "CallSite table length = " << CallSiteTableLength << '\n';
BC.outs() << '\n';
}

this->HasEHRanges = CallSitePtr < CallSiteTableEnd;
Expand All @@ -185,12 +186,13 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
LandingPad += LPStart;

if (opts::PrintExceptions) {
outs() << "Call Site: [0x" << Twine::utohexstr(RangeBase + Start)
<< ", 0x" << Twine::utohexstr(RangeBase + Start + Length)
<< "); landing pad: 0x" << Twine::utohexstr(LandingPad)
<< "; action entry: 0x" << Twine::utohexstr(ActionEntry) << "\n";
outs() << " current offset is " << (CallSitePtr - CallSiteTableStart)
<< '\n';
BC.outs() << "Call Site: [0x" << Twine::utohexstr(RangeBase + Start)
<< ", 0x" << Twine::utohexstr(RangeBase + Start + Length)
<< "); landing pad: 0x" << Twine::utohexstr(LandingPad)
<< "; action entry: 0x" << Twine::utohexstr(ActionEntry)
<< "\n";
BC.outs() << " current offset is " << (CallSitePtr - CallSiteTableStart)
<< '\n';
}

// Create a handler entry if necessary.
Expand All @@ -209,15 +211,16 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
"BOLT-ERROR: cannot have landing pads in different functions");
setHasIndirectTargetToSplitFragment(true);
BC.addFragmentsToSkip(this);
return;
return Error::success();
}

const uint64_t LPOffset = LandingPad - getAddress();
if (!getInstructionAtOffset(LPOffset)) {
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: landing pad " << Twine::utohexstr(LPOffset)
<< " not pointing to an instruction in function " << *this
<< " - ignoring.\n";
BC.errs() << "BOLT-WARNING: landing pad "
<< Twine::utohexstr(LPOffset)
<< " not pointing to an instruction in function " << *this
<< " - ignoring.\n";
} else {
auto Label = Labels.find(LPOffset);
if (Label != Labels.end()) {
Expand Down Expand Up @@ -271,7 +274,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
OS << "0x" << Twine::utohexstr(TypeAddress);
};
if (opts::PrintExceptions)
outs() << " actions: ";
BC.outs() << " actions: ";
uint64_t ActionPtr = ActionTableStart + ActionEntry - 1;
int64_t ActionType;
int64_t ActionNext;
Expand All @@ -281,21 +284,21 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
const uint32_t Self = ActionPtr;
ActionNext = Data.getSLEB128(&ActionPtr);
if (opts::PrintExceptions)
outs() << Sep << "(" << ActionType << ", " << ActionNext << ") ";
BC.outs() << Sep << "(" << ActionType << ", " << ActionNext << ") ";
if (ActionType == 0) {
if (opts::PrintExceptions)
outs() << "cleanup";
BC.outs() << "cleanup";
} else if (ActionType > 0) {
// It's an index into a type table.
MaxTypeIndex =
std::max(MaxTypeIndex, static_cast<unsigned>(ActionType));
if (opts::PrintExceptions) {
outs() << "catch type ";
printType(ActionType, outs());
BC.outs() << "catch type ";
printType(ActionType, BC.outs());
}
} else { // ActionType < 0
if (opts::PrintExceptions)
outs() << "filter exception types ";
BC.outs() << "filter exception types ";
const char *TSep = "";
// ActionType is a negative *byte* offset into *uleb128-encoded* table
// of indices with base 1.
Expand All @@ -305,8 +308,8 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
while (uint64_t Index = Data.getULEB128(&TypeIndexTablePtr)) {
MaxTypeIndex = std::max(MaxTypeIndex, static_cast<unsigned>(Index));
if (opts::PrintExceptions) {
outs() << TSep;
printType(Index, outs());
BC.outs() << TSep;
printType(Index, BC.outs());
TSep = ", ";
}
}
Expand All @@ -319,11 +322,11 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
ActionPtr = Self + ActionNext;
} while (ActionNext);
if (opts::PrintExceptions)
outs() << '\n';
BC.outs() << '\n';
}
}
if (opts::PrintExceptions)
outs() << '\n';
BC.outs() << '\n';

assert(TypeIndexTableStart + MaxTypeIndexTableOffset <=
Data.getData().size() &&
Expand Down Expand Up @@ -354,6 +357,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
LSDATypeIndexTable =
LSDASectionData.slice(TypeIndexTableStart, MaxTypeIndexTableOffset);
}
return Error::success();
}

void BinaryFunction::updateEHRanges() {
Expand Down Expand Up @@ -460,7 +464,9 @@ void BinaryFunction::updateEHRanges() {

const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;

CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) {
CFIReaderWriter::CFIReaderWriter(BinaryContext &BC,
const DWARFDebugFrame &EHFrame)
: BC(BC) {
// Prepare FDEs for fast lookup
for (const dwarf::FrameEntry &Entry : EHFrame.entries()) {
const auto *CurFDE = dyn_cast<dwarf::FDE>(&Entry);
Expand All @@ -475,10 +481,10 @@ CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) {
if (FDEI->second->getAddressRange() == 0) {
FDEI->second = CurFDE;
} else if (opts::Verbosity > 0) {
errs() << "BOLT-WARNING: different FDEs for function at 0x"
<< Twine::utohexstr(FDEI->first)
<< " detected; sizes: " << FDEI->second->getAddressRange()
<< " and " << CurFDE->getAddressRange() << '\n';
BC.errs() << "BOLT-WARNING: different FDEs for function at 0x"
<< Twine::utohexstr(FDEI->first)
<< " detected; sizes: " << FDEI->second->getAddressRange()
<< " and " << CurFDE->getAddressRange() << '\n';
}
}
} else {
Expand Down Expand Up @@ -508,8 +514,8 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
*CurFDE.getLinkedCIE()->getPersonalityEncoding());
}

auto decodeFrameInstruction = [&Function, &Offset, Address, CodeAlignment,
DataAlignment](
auto decodeFrameInstruction = [this, &Function, &Offset, Address,
CodeAlignment, DataAlignment](
const CFIProgram::Instruction &Instr) {
uint8_t Opcode = Instr.Opcode;
if (Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK)
Expand Down Expand Up @@ -601,7 +607,7 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
case DW_CFA_val_offset_sf:
case DW_CFA_val_offset:
if (opts::Verbosity >= 1) {
errs() << "BOLT-WARNING: DWARF val_offset() unimplemented\n";
BC.errs() << "BOLT-WARNING: DWARF val_offset() unimplemented\n";
}
return false;
case DW_CFA_def_cfa_expression:
Expand All @@ -622,7 +628,7 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
}
case DW_CFA_MIPS_advance_loc8:
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: DW_CFA_MIPS_advance_loc unimplemented\n";
BC.errs() << "BOLT-WARNING: DW_CFA_MIPS_advance_loc unimplemented\n";
return false;
case DW_CFA_GNU_window_save:
// DW_CFA_GNU_window_save and DW_CFA_GNU_NegateRAState just use the same
Expand All @@ -633,17 +639,17 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
break;
}
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: DW_CFA_GNU_window_save unimplemented\n";
BC.errs() << "BOLT-WARNING: DW_CFA_GNU_window_save unimplemented\n";
return false;
case DW_CFA_lo_user:
case DW_CFA_hi_user:
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: DW_CFA_*_user unimplemented\n";
BC.errs() << "BOLT-WARNING: DW_CFA_*_user unimplemented\n";
return false;
default:
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: Unrecognized CFI instruction: " << Instr.Opcode
<< '\n';
BC.errs() << "BOLT-WARNING: Unrecognized CFI instruction: "
<< Instr.Opcode << '\n';
return false;
}

Expand Down
5 changes: 3 additions & 2 deletions bolt/lib/Core/ParallelUtilities.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@ inline unsigned estimateTotalCost(const BinaryContext &BC,

// Switch to trivial scheduling if total estimated work is zero
if (TotalCost == 0) {
outs() << "BOLT-WARNING: Running parallel work of 0 estimated cost, will "
"switch to trivial scheduling.\n";
BC.outs()
<< "BOLT-WARNING: Running parallel work of 0 estimated cost, will "
"switch to trivial scheduling.\n";

SchedPolicy = SP_TRIVIAL;
TotalCost = BC.getBinaryFunctions().size();
Expand Down
14 changes: 8 additions & 6 deletions bolt/lib/Passes/ADRRelaxationPass.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,10 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) {
// invalidate this offset, so we have to rely on linker-inserted NOP to
// replace it with ADRP, and abort if it is not present.
auto L = BC.scopeLock();
errs() << formatv("BOLT-ERROR: Cannot relax adr in non-simple function "
"{0}. Use --strict option to override\n",
BF.getOneName());
BC.errs() << formatv(
"BOLT-ERROR: Cannot relax adr in non-simple function "
"{0}. Use --strict option to override\n",
BF.getOneName());
PassFailed = true;
return;
}
Expand All @@ -97,9 +98,9 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) {
}
}

void ADRRelaxationPass::runOnFunctions(BinaryContext &BC) {
Error ADRRelaxationPass::runOnFunctions(BinaryContext &BC) {
if (!opts::AdrPassOpt || !BC.HasRelocations)
return;
return Error::success();

ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
runOnFunction(BF);
Expand All @@ -110,7 +111,8 @@ void ADRRelaxationPass::runOnFunctions(BinaryContext &BC) {
"ADRRelaxationPass");

if (PassFailed)
exit(1);
return createFatalBOLTError("");
return Error::success();
}

} // end namespace bolt
Expand Down
5 changes: 3 additions & 2 deletions bolt/lib/Passes/Aligner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,9 @@ void AlignerPass::alignBlocks(BinaryFunction &Function,
}
}

void AlignerPass::runOnFunctions(BinaryContext &BC) {
Error AlignerPass::runOnFunctions(BinaryContext &BC) {
if (!BC.HasRelocations)
return;
return Error::success();

AlignHistogram.resize(opts::BlockAlignment);

Expand Down Expand Up @@ -179,6 +179,7 @@ void AlignerPass::runOnFunctions(BinaryContext &BC) {
dbgs() << "BOLT-DEBUG: total execution count of aligned blocks: "
<< AlignedBlocksCount << '\n';
);
return Error::success();
}

} // end namespace bolt
Expand Down
11 changes: 6 additions & 5 deletions bolt/lib/Passes/AllocCombiner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,17 +103,18 @@ void AllocCombinerPass::combineAdjustments(BinaryFunction &BF) {
}
}

void AllocCombinerPass::runOnFunctions(BinaryContext &BC) {
Error AllocCombinerPass::runOnFunctions(BinaryContext &BC) {
if (opts::FrameOptimization == FOP_NONE)
return;
return Error::success();

runForAllWeCare(BC.getBinaryFunctions(), [&](BinaryFunction &Function) {
combineAdjustments(Function);
});

outs() << "BOLT-INFO: Allocation combiner: " << NumCombined
<< " empty spaces coalesced (dyn count: " << DynamicCountCombined
<< ").\n";
BC.outs() << "BOLT-INFO: Allocation combiner: " << NumCombined
<< " empty spaces coalesced (dyn count: " << DynamicCountCombined
<< ").\n";
return Error::success();
}

} // end namespace bolt
Expand Down
19 changes: 10 additions & 9 deletions bolt/lib/Passes/AsmDump.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ void dumpCFI(const BinaryFunction &BF, const MCInst &Instr, AsmPrinter &MAP) {
case MCCFIInstruction::OpRememberState:
case MCCFIInstruction::OpRestoreState:
if (opts::Verbosity >= 2)
errs()
BF.getBinaryContext().errs()
<< "BOLT-WARNING: AsmDump: skipping unsupported CFI instruction in "
<< BF << ".\n";

Expand Down Expand Up @@ -102,9 +102,9 @@ void dumpFunction(const BinaryFunction &BF) {
// Make sure the new directory exists, creating it if necessary.
if (!opts::AsmDump.empty()) {
if (std::error_code EC = sys::fs::create_directories(opts::AsmDump)) {
errs() << "BOLT-ERROR: could not create directory '" << opts::AsmDump
<< "': " << EC.message() << '\n';
exit(1);
BC.errs() << "BOLT-ERROR: could not create directory '" << opts::AsmDump
<< "': " << EC.message() << '\n';
return;
}
}

Expand All @@ -115,14 +115,14 @@ void dumpFunction(const BinaryFunction &BF) {
? (PrintName + ".s")
: (opts::AsmDump + sys::path::get_separator() + PrintName + ".s")
.str();
outs() << "BOLT-INFO: Dumping function assembly to " << Filename << "\n";
BC.outs() << "BOLT-INFO: Dumping function assembly to " << Filename << "\n";

std::error_code EC;
raw_fd_ostream OS(Filename, EC, sys::fs::OF_None);
if (EC) {
errs() << "BOLT-ERROR: " << EC.message() << ", unable to open " << Filename
<< " for output.\n";
exit(1);
BC.errs() << "BOLT-ERROR: " << EC.message() << ", unable to open "
<< Filename << " for output.\n";
return;
}
OS.SetUnbuffered();

Expand Down Expand Up @@ -237,9 +237,10 @@ void dumpFunction(const BinaryFunction &BF) {
dumpBinaryDataSymbols(OS, BD, LastSection);
}

void AsmDumpPass::runOnFunctions(BinaryContext &BC) {
Error AsmDumpPass::runOnFunctions(BinaryContext &BC) {
for (const auto &BFIt : BC.getBinaryFunctions())
dumpFunction(BFIt.second);
return Error::success();
}

} // namespace bolt
Expand Down
14 changes: 7 additions & 7 deletions bolt/lib/Passes/BinaryFunctionCallGraph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -278,13 +278,13 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
bool PrintInfo = false;
#endif
if (PrintInfo || opts::Verbosity > 0)
outs() << format("BOLT-INFO: buildCallGraph: %u nodes, %u callsites "
"(%u recursive), density = %.6lf, %u callsites not "
"processed, %u callsites with invalid profile, "
"used perf data for %u stale functions.\n",
Cg.numNodes(), TotalCallsites, RecursiveCallsites,
Cg.density(), NotProcessed, NoProfileCallsites,
NumFallbacks);
BC.outs() << format("BOLT-INFO: buildCallGraph: %u nodes, %u callsites "
"(%u recursive), density = %.6lf, %u callsites not "
"processed, %u callsites with invalid profile, "
"used perf data for %u stale functions.\n",
Cg.numNodes(), TotalCallsites, RecursiveCallsites,
Cg.density(), NotProcessed, NoProfileCallsites,
NumFallbacks);

if (opts::DumpCGDot.getNumOccurrences()) {
Cg.printDot(opts::DumpCGDot, [&](CallGraph::NodeId Id) {
Expand Down
379 changes: 205 additions & 174 deletions bolt/lib/Passes/BinaryPasses.cpp

Large diffs are not rendered by default.

27 changes: 14 additions & 13 deletions bolt/lib/Passes/CMOVConversion.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,14 +168,14 @@ int calculateConditionBias(const BinaryBasicBlock &BB,
return -1;
}

void CMOVConversion::Stats::dump() {
outs() << "converted static " << StaticPerformed << "/" << StaticPossible
<< formatv(" ({0:P}) ", getStaticRatio())
<< "hammock(s) into CMOV sequences, with dynamic execution count "
<< DynamicPerformed << "/" << DynamicPossible
<< formatv(" ({0:P}), ", getDynamicRatio()) << "saving " << RemovedMP
<< "/" << PossibleMP << formatv(" ({0:P}) ", getMPRatio())
<< "mispredictions\n";
void CMOVConversion::Stats::dumpTo(raw_ostream &OS) {
OS << "converted static " << StaticPerformed << "/" << StaticPossible
<< formatv(" ({0:P}) ", getStaticRatio())
<< "hammock(s) into CMOV sequences, with dynamic execution count "
<< DynamicPerformed << "/" << DynamicPossible
<< formatv(" ({0:P}), ", getDynamicRatio()) << "saving " << RemovedMP
<< "/" << PossibleMP << formatv(" ({0:P}) ", getMPRatio())
<< "mispredictions\n";
}

void CMOVConversion::runOnFunction(BinaryFunction &Function) {
Expand Down Expand Up @@ -265,22 +265,23 @@ void CMOVConversion::runOnFunction(BinaryFunction &Function) {
if (Modified)
Function.eraseInvalidBBs();
if (opts::Verbosity > 1) {
outs() << "BOLT-INFO: CMOVConversion: " << Function << ", ";
Local.dump();
BC.outs() << "BOLT-INFO: CMOVConversion: " << Function << ", ";
Local.dumpTo(BC.outs());
}
Global = Global + Local;
}

void CMOVConversion::runOnFunctions(BinaryContext &BC) {
Error CMOVConversion::runOnFunctions(BinaryContext &BC) {
for (auto &It : BC.getBinaryFunctions()) {
BinaryFunction &Function = It.second;
if (!shouldOptimize(Function))
continue;
runOnFunction(Function);
}

outs() << "BOLT-INFO: CMOVConversion total: ";
Global.dump();
BC.outs() << "BOLT-INFO: CMOVConversion total: ";
Global.dumpTo(BC.outs());
return Error::success();
}

} // end namespace bolt
Expand Down
39 changes: 20 additions & 19 deletions bolt/lib/Passes/CacheMetrics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,8 @@ double expectedCacheHitRatio(

} // namespace

void CacheMetrics::printAll(const std::vector<BinaryFunction *> &BFs) {
void CacheMetrics::printAll(raw_ostream &OS,
const std::vector<BinaryFunction *> &BFs) {
// Stats related to hot-cold code splitting
size_t NumFunctions = 0;
size_t NumProfiledFunctions = 0;
Expand Down Expand Up @@ -222,36 +223,36 @@ void CacheMetrics::printAll(const std::vector<BinaryFunction *> &BFs) {
}
}

outs() << format(" There are %zu functions;", NumFunctions)
<< format(" %zu (%.2lf%%) are in the hot section,", NumHotFunctions,
100.0 * NumHotFunctions / NumFunctions)
<< format(" %zu (%.2lf%%) have profile\n", NumProfiledFunctions,
100.0 * NumProfiledFunctions / NumFunctions);
outs() << format(" There are %zu basic blocks;", NumBlocks)
<< format(" %zu (%.2lf%%) are in the hot section\n", NumHotBlocks,
100.0 * NumHotBlocks / NumBlocks);
OS << format(" There are %zu functions;", NumFunctions)
<< format(" %zu (%.2lf%%) are in the hot section,", NumHotFunctions,
100.0 * NumHotFunctions / NumFunctions)
<< format(" %zu (%.2lf%%) have profile\n", NumProfiledFunctions,
100.0 * NumProfiledFunctions / NumFunctions);
OS << format(" There are %zu basic blocks;", NumBlocks)
<< format(" %zu (%.2lf%%) are in the hot section\n", NumHotBlocks,
100.0 * NumHotBlocks / NumBlocks);

assert(TotalCodeMinAddr <= TotalCodeMaxAddr && "incorrect output addresses");
size_t HotCodeSize = HotCodeMaxAddr - HotCodeMinAddr;
size_t TotalCodeSize = TotalCodeMaxAddr - TotalCodeMinAddr;

size_t HugePage2MB = 2 << 20;
outs() << format(" Hot code takes %.2lf%% of binary (%zu bytes out of %zu, "
"%.2lf huge pages)\n",
100.0 * HotCodeSize / TotalCodeSize, HotCodeSize,
TotalCodeSize, double(HotCodeSize) / HugePage2MB);
OS << format(" Hot code takes %.2lf%% of binary (%zu bytes out of %zu, "
"%.2lf huge pages)\n",
100.0 * HotCodeSize / TotalCodeSize, HotCodeSize, TotalCodeSize,
double(HotCodeSize) / HugePage2MB);

// Stats related to expected cache performance
std::unordered_map<BinaryBasicBlock *, uint64_t> BBAddr;
std::unordered_map<BinaryBasicBlock *, uint64_t> BBSize;
extractBasicBlockInfo(BFs, BBAddr, BBSize);

outs() << " Expected i-TLB cache hit ratio: "
<< format("%.2lf%%\n", expectedCacheHitRatio(BFs, BBAddr, BBSize));
OS << " Expected i-TLB cache hit ratio: "
<< format("%.2lf%%\n", expectedCacheHitRatio(BFs, BBAddr, BBSize));

auto Stats = calcTSPScore(BFs, BBAddr, BBSize);
outs() << " TSP score: "
<< format("%.2lf%% (%zu out of %zu)\n",
100.0 * Stats.first / std::max<uint64_t>(Stats.second, 1),
Stats.first, Stats.second);
OS << " TSP score: "
<< format("%.2lf%% (%zu out of %zu)\n",
100.0 * Stats.first / std::max<uint64_t>(Stats.second, 1),
Stats.first, Stats.second);
}
6 changes: 4 additions & 2 deletions bolt/lib/Passes/FixRISCVCallsPass.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) {
}
}

void FixRISCVCallsPass::runOnFunctions(BinaryContext &BC) {
Error FixRISCVCallsPass::runOnFunctions(BinaryContext &BC) {
if (!BC.isRISCV() || !BC.HasRelocations)
return;
return Error::success();

ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
runOnFunction(BF);
Expand All @@ -79,6 +79,8 @@ void FixRISCVCallsPass::runOnFunctions(BinaryContext &BC) {
ParallelUtilities::runOnEachFunction(
BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, nullptr,
"FixRISCVCalls");

return Error::success();
}

} // namespace bolt
Expand Down
5 changes: 3 additions & 2 deletions bolt/lib/Passes/FixRelaxationPass.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ void FixRelaxations::runOnFunction(BinaryFunction &BF) {
}
}

void FixRelaxations::runOnFunctions(BinaryContext &BC) {
Error FixRelaxations::runOnFunctions(BinaryContext &BC) {
if (!BC.isAArch64() || !BC.HasRelocations)
return;
return Error::success();

ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
runOnFunction(BF);
Expand All @@ -58,6 +58,7 @@ void FixRelaxations::runOnFunctions(BinaryContext &BC) {
ParallelUtilities::runOnEachFunction(
BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, nullptr,
"FixRelaxations");
return Error::success();
}

} // namespace bolt
Expand Down
15 changes: 8 additions & 7 deletions bolt/lib/Passes/FrameAnalysis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ class FrameAccessAnalysis {
if (IsIndexed || (!FIE.Size && (FIE.IsLoad || FIE.IsStore))) {
LLVM_DEBUG(dbgs() << "Giving up on indexed memory access/unknown size\n");
LLVM_DEBUG(dbgs() << "Blame insn: ");
LLVM_DEBUG(BC.printInstruction(outs(), Inst, 0, &BF, true, false, false));
LLVM_DEBUG(BC.printInstruction(dbgs(), Inst, 0, &BF, true, false, false));
LLVM_DEBUG(Inst.dump());
return false;
}
Expand Down Expand Up @@ -570,13 +570,14 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
}

void FrameAnalysis::printStats() {
outs() << "BOLT-INFO: FRAME ANALYSIS: " << NumFunctionsNotOptimized
<< " function(s) were not optimized.\n"
<< "BOLT-INFO: FRAME ANALYSIS: " << NumFunctionsFailedRestoreFI
<< " function(s) "
<< format("(%.1lf%% dyn cov)",
BC.outs() << "BOLT-INFO: FRAME ANALYSIS: " << NumFunctionsNotOptimized
<< " function(s) were not optimized.\n"
<< "BOLT-INFO: FRAME ANALYSIS: " << NumFunctionsFailedRestoreFI
<< " function(s) "
<< format(
"(%.1lf%% dyn cov)",
(100.0 * CountFunctionsFailedRestoreFI / CountDenominator))
<< " could not have its frame indices restored.\n";
<< " could not have its frame indices restored.\n";
}

void FrameAnalysis::clearSPTMap() {
Expand Down
58 changes: 35 additions & 23 deletions bolt/lib/Passes/FrameOptimizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -221,9 +221,9 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
LLVM_DEBUG(dbgs() << "FOP modified \"" << BF.getPrintName() << "\"\n");
}

void FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
Error FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
if (opts::FrameOptimization == FOP_NONE)
return;
return Error::success();

std::unique_ptr<BinaryFunctionCallGraph> CG;
std::unique_ptr<FrameAnalysis> FA;
Expand Down Expand Up @@ -285,29 +285,31 @@ void FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
{
NamedRegionTimer T1("shrinkwrapping", "shrink wrapping", "FOP",
"FOP breakdown", opts::TimeOpts);
performShrinkWrapping(*RA, *FA, BC);
if (Error E = performShrinkWrapping(*RA, *FA, BC))
return Error(std::move(E));
}

outs() << "BOLT-INFO: FOP optimized " << NumRedundantLoads
<< " redundant load(s) and " << NumRedundantStores
<< " unused store(s)\n";
outs() << "BOLT-INFO: Frequency of redundant loads is " << FreqRedundantLoads
<< " and frequency of unused stores is " << FreqRedundantStores
<< "\n";
outs() << "BOLT-INFO: Frequency of loads changed to use a register is "
<< FreqLoadsChangedToReg
<< " and frequency of loads changed to use an immediate is "
<< FreqLoadsChangedToImm << "\n";
outs() << "BOLT-INFO: FOP deleted " << NumLoadsDeleted
<< " load(s) (dyn count: " << FreqLoadsDeleted << ") and "
<< NumRedundantStores << " store(s)\n";
BC.outs() << "BOLT-INFO: FOP optimized " << NumRedundantLoads
<< " redundant load(s) and " << NumRedundantStores
<< " unused store(s)\n";
BC.outs() << "BOLT-INFO: Frequency of redundant loads is "
<< FreqRedundantLoads << " and frequency of unused stores is "
<< FreqRedundantStores << "\n";
BC.outs() << "BOLT-INFO: Frequency of loads changed to use a register is "
<< FreqLoadsChangedToReg
<< " and frequency of loads changed to use an immediate is "
<< FreqLoadsChangedToImm << "\n";
BC.outs() << "BOLT-INFO: FOP deleted " << NumLoadsDeleted
<< " load(s) (dyn count: " << FreqLoadsDeleted << ") and "
<< NumRedundantStores << " store(s)\n";
FA->printStats();
ShrinkWrapping::printStats();
ShrinkWrapping::printStats(BC);
return Error::success();
}

void FrameOptimizerPass::performShrinkWrapping(const RegAnalysis &RA,
const FrameAnalysis &FA,
BinaryContext &BC) {
Error FrameOptimizerPass::performShrinkWrapping(const RegAnalysis &RA,
const FrameAnalysis &FA,
BinaryContext &BC) {
// Initialize necessary annotations to allow safe parallel accesses to
// annotation index in MIB
BC.MIB->getOrCreateAnnotationIndex(CalleeSavedAnalysis::getSaveTagName());
Expand Down Expand Up @@ -357,12 +359,21 @@ void FrameOptimizerPass::performShrinkWrapping(const RegAnalysis &RA,

const bool HotOnly = opts::FrameOptimization == FOP_HOT;

Error SWError = Error::success();

ParallelUtilities::WorkFuncWithAllocTy WorkFunction =
[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocatorId) {
DataflowInfoManager Info(BF, &RA, &FA, AllocatorId);
ShrinkWrapping SW(FA, BF, Info, AllocatorId);

if (SW.perform(HotOnly)) {
auto ChangedOrErr = SW.perform(HotOnly);
if (auto E = ChangedOrErr.takeError()) {
std::lock_guard<std::mutex> Lock(FuncsChangedMutex);
SWError = joinErrors(std::move(SWError), Error(std::move(E)));
return;
}
const bool Changed = *ChangedOrErr;
if (Changed) {
std::lock_guard<std::mutex> Lock(FuncsChangedMutex);
FuncsChanged.insert(&BF);
LLVM_DEBUG(LogFunc(BF));
Expand All @@ -374,10 +385,11 @@ void FrameOptimizerPass::performShrinkWrapping(const RegAnalysis &RA,
SkipPredicate, "shrink-wrapping");

if (!Top10Funcs.empty()) {
outs() << "BOLT-INFO: top 10 functions changed by shrink wrapping:\n";
BC.outs() << "BOLT-INFO: top 10 functions changed by shrink wrapping:\n";
for (const auto &Elmt : Top10Funcs)
outs() << Elmt.first << " : " << Elmt.second->getPrintName() << "\n";
BC.outs() << Elmt.first << " : " << Elmt.second->getPrintName() << "\n";
}
return SWError;
}

} // namespace bolt
Expand Down
5 changes: 3 additions & 2 deletions bolt/lib/Passes/Hugify.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ using namespace llvm;
namespace llvm {
namespace bolt {

void HugePage::runOnFunctions(BinaryContext &BC) {
Error HugePage::runOnFunctions(BinaryContext &BC) {
auto *RtLibrary = BC.getRuntimeLibrary();
if (!RtLibrary || !BC.isELF() || !BC.StartFunctionAddress) {
return;
return Error::success();
}

auto createSimpleFunction =
Expand All @@ -45,6 +45,7 @@ void HugePage::runOnFunctions(BinaryContext &BC) {
const MCSymbol *StartSym = Start->getSymbol();
createSimpleFunction("__bolt_hugify_start_program",
BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));
return Error::success();
}
} // namespace bolt
} // namespace llvm
20 changes: 11 additions & 9 deletions bolt/lib/Passes/IdenticalCodeFolding.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ typedef std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
namespace llvm {
namespace bolt {

void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
Error IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
const size_t OriginalFunctionCount = BC.getBinaryFunctions().size();
uint64_t NumFunctionsFolded = 0;
std::atomic<uint64_t> NumJTFunctionsFolded{0};
Expand Down Expand Up @@ -508,14 +508,16 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
});

if (NumFunctionsFolded)
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded << " out of "
<< OriginalFunctionCount << " functions in " << Iteration
<< " passes. " << NumJTFunctionsFolded
<< " functions had jump tables.\n"
<< "BOLT-INFO: Removing all identical functions will save "
<< format("%.2lf", (double)BytesSavedEstimate / 1024)
<< " KB of code space. Folded functions were called " << NumCalled
<< " times based on profile.\n";
BC.outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded << " out of "
<< OriginalFunctionCount << " functions in " << Iteration
<< " passes. " << NumJTFunctionsFolded
<< " functions had jump tables.\n"
<< "BOLT-INFO: Removing all identical functions will save "
<< format("%.2lf", (double)BytesSavedEstimate / 1024)
<< " KB of code space. Folded functions were called " << NumCalled
<< " times based on profile.\n";

return Error::success();
}

} // namespace bolt
Expand Down
201 changes: 103 additions & 98 deletions bolt/lib/Passes/IndirectCallPromotion.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,10 @@ static bool verifyProfile(std::map<uint64_t, BinaryFunction> &BFs) {
if (BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && BI->Count > 0) {
if (BB.getKnownExecutionCount() == 0 ||
SuccBB->getKnownExecutionCount() == 0) {
errs() << "BOLT-WARNING: profile verification failed after ICP for "
"function "
<< BF << '\n';
BF.getBinaryContext().errs()
<< "BOLT-WARNING: profile verification failed after ICP for "
"function "
<< BF << '\n';
IsValid = false;
}
}
Expand Down Expand Up @@ -526,6 +527,7 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets,
size_t &N, BinaryBasicBlock &BB,
MCInst &CallInst,
MCInst *&TargetFetchInst) const {
const BinaryContext &BC = BB.getFunction()->getBinaryContext();
const JumpTable *JT = BB.getFunction()->getJumpTable(CallInst);
SymTargetsType SymTargets;

Expand Down Expand Up @@ -556,8 +558,9 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets,
if (!HotTargets.empty()) {
if (opts::Verbosity >= 1)
for (size_t I = 0; I < HotTargets.size(); ++I)
outs() << "BOLT-INFO: HotTarget[" << I << "] = (" << HotTargets[I].first
<< ", " << HotTargets[I].second << ")\n";
BC.outs() << "BOLT-INFO: HotTarget[" << I << "] = ("
<< HotTargets[I].first << ", " << HotTargets[I].second
<< ")\n";

// Recompute hottest targets, now discriminating which index is hot
// NOTE: This is a tradeoff. On one hand, we get index information. On the
Expand Down Expand Up @@ -611,9 +614,9 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets,
N = I;

if (N == 0 && opts::Verbosity >= 1) {
outs() << "BOLT-INFO: ICP failed in " << *BB.getFunction() << " in "
<< BB.getName() << ": failed to meet thresholds after memory "
<< "profile data was loaded.\n";
BC.outs() << "BOLT-INFO: ICP failed in " << *BB.getFunction() << " in "
<< BB.getName() << ": failed to meet thresholds after memory "
<< "profile data was loaded.\n";
return SymTargets;
}
}
Expand Down Expand Up @@ -974,9 +977,9 @@ size_t IndirectCallPromotion::canPromoteCallsite(
if (Targets.empty() || !NumCalls) {
if (opts::Verbosity >= 1) {
const ptrdiff_t InstIdx = &Inst - &(*BB.begin());
outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx << " in "
<< BB.getName() << ", calls = " << NumCalls
<< ", targets empty or NumCalls == 0.\n";
BC.outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx
<< " in " << BB.getName() << ", calls = " << NumCalls
<< ", targets empty or NumCalls == 0.\n";
}
return 0;
}
Expand Down Expand Up @@ -1015,10 +1018,10 @@ size_t IndirectCallPromotion::canPromoteCallsite(
if (TopNFrequency == 0 || TopNFrequency < opts::ICPMispredictThreshold) {
if (opts::Verbosity >= 1) {
const ptrdiff_t InstIdx = &Inst - &(*BB.begin());
outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx
<< " in " << BB.getName() << ", calls = " << NumCalls
<< ", top N mis. frequency " << format("%.1f", TopNFrequency)
<< "% < " << opts::ICPMispredictThreshold << "%\n";
BC.outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx
<< " in " << BB.getName() << ", calls = " << NumCalls
<< ", top N mis. frequency " << format("%.1f", TopNFrequency)
<< "% < " << opts::ICPMispredictThreshold << "%\n";
}
return 0;
}
Expand Down Expand Up @@ -1061,11 +1064,11 @@ size_t IndirectCallPromotion::canPromoteCallsite(
if (TopNMispredictFrequency < opts::ICPMispredictThreshold) {
if (opts::Verbosity >= 1) {
const ptrdiff_t InstIdx = &Inst - &(*BB.begin());
outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx
<< " in " << BB.getName() << ", calls = " << NumCalls
<< ", top N mispredict frequency "
<< format("%.1f", TopNMispredictFrequency) << "% < "
<< opts::ICPMispredictThreshold << "%\n";
BC.outs() << "BOLT-INFO: ICP failed in " << *BF << " @ " << InstIdx
<< " in " << BB.getName() << ", calls = " << NumCalls
<< ", top N mispredict frequency "
<< format("%.1f", TopNMispredictFrequency) << "% < "
<< opts::ICPMispredictThreshold << "%\n";
}
return 0;
}
Expand Down Expand Up @@ -1106,29 +1109,29 @@ void IndirectCallPromotion::printCallsiteInfo(
const bool IsJumpTable = BB.getFunction()->getJumpTable(Inst);
const ptrdiff_t InstIdx = &Inst - &(*BB.begin());

outs() << "BOLT-INFO: ICP candidate branch info: " << *BB.getFunction()
<< " @ " << InstIdx << " in " << BB.getName()
<< " -> calls = " << NumCalls
<< (IsTailCall ? " (tail)" : (IsJumpTable ? " (jump table)" : ""))
<< "\n";
BC.outs() << "BOLT-INFO: ICP candidate branch info: " << *BB.getFunction()
<< " @ " << InstIdx << " in " << BB.getName()
<< " -> calls = " << NumCalls
<< (IsTailCall ? " (tail)" : (IsJumpTable ? " (jump table)" : ""))
<< "\n";
for (size_t I = 0; I < N; I++) {
const double Frequency = 100.0 * Targets[I].Branches / NumCalls;
const double MisFrequency = 100.0 * Targets[I].Mispreds / NumCalls;
outs() << "BOLT-INFO: ";
BC.outs() << "BOLT-INFO: ";
if (Targets[I].To.Sym)
outs() << Targets[I].To.Sym->getName();
BC.outs() << Targets[I].To.Sym->getName();
else
outs() << Targets[I].To.Addr;
outs() << ", calls = " << Targets[I].Branches
<< ", mispreds = " << Targets[I].Mispreds
<< ", taken freq = " << format("%.1f", Frequency) << "%"
<< ", mis. freq = " << format("%.1f", MisFrequency) << "%";
BC.outs() << Targets[I].To.Addr;
BC.outs() << ", calls = " << Targets[I].Branches
<< ", mispreds = " << Targets[I].Mispreds
<< ", taken freq = " << format("%.1f", Frequency) << "%"
<< ", mis. freq = " << format("%.1f", MisFrequency) << "%";
bool First = true;
for (uint64_t JTIndex : Targets[I].JTIndices) {
outs() << (First ? ", indices = " : ", ") << JTIndex;
BC.outs() << (First ? ", indices = " : ", ") << JTIndex;
First = false;
}
outs() << "\n";
BC.outs() << "\n";
}

LLVM_DEBUG({
Expand All @@ -1137,9 +1140,9 @@ void IndirectCallPromotion::printCallsiteInfo(
});
}

void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
Error IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
if (opts::ICP == ICP_NONE)
return;
return Error::success();

auto &BFs = BC.getBinaryFunctions();

Expand Down Expand Up @@ -1222,9 +1225,9 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
Functions.insert(std::get<2>(IC));
++Num;
}
outs() << "BOLT-INFO: ICP Total indirect calls = " << TotalIndirectCalls
<< ", " << Num << " callsites cover " << opts::ICPTopCallsites
<< "% of all indirect calls\n";
BC.outs() << "BOLT-INFO: ICP Total indirect calls = " << TotalIndirectCalls
<< ", " << Num << " callsites cover " << opts::ICPTopCallsites
<< "% of all indirect calls\n";
}

for (BinaryFunction *FuncPtr : Functions) {
Expand Down Expand Up @@ -1301,11 +1304,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
Info.getLivenessAnalysis().getStateBefore(Inst);
if (!State || (State && (*State)[BC.MIB->getFlagsReg()])) {
if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< (State ? ", cannot clobber flags reg.\n"
: ", no liveness data available.\n");
BC.outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< (State ? ", cannot clobber flags reg.\n"
: ", no liveness data available.\n");
continue;
}
}
Expand Down Expand Up @@ -1341,11 +1344,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
if (SymTargets.size() < N) {
const size_t LastTarget = SymTargets.size();
if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< ", ICP failed to find target symbol for "
<< Targets[LastTarget].To.Sym->getName() << "\n";
BC.outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< ", ICP failed to find target symbol for "
<< Targets[LastTarget].To.Sym->getName() << "\n";
continue;
}

Expand Down Expand Up @@ -1374,10 +1377,10 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {

if (ICPcode.empty()) {
if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< ", unable to generate promoted call code.\n";
BC.outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< ", calls = " << NumCalls
<< ", unable to generate promoted call code.\n";
continue;
}

Expand Down Expand Up @@ -1410,9 +1413,9 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
BBs.push_back(MergeBlock);

if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: ICP succeeded in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< " -> calls = " << NumCalls << "\n";
BC.outs() << "BOLT-INFO: ICP succeeded in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
<< " -> calls = " << NumCalls << "\n";

if (IsJumpTable)
++TotalOptimizedJumpTableCallsites;
Expand All @@ -1426,52 +1429,54 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
TotalIndirectJmps += FuncTotalIndirectJmps;
}

outs() << "BOLT-INFO: ICP total indirect callsites with profile = "
<< TotalIndirectCallsites << "\n"
<< "BOLT-INFO: ICP total jump table callsites = "
<< TotalJumpTableCallsites << "\n"
<< "BOLT-INFO: ICP total number of calls = " << TotalCalls << "\n"
<< "BOLT-INFO: ICP percentage of calls that are indirect = "
<< format("%.1f", (100.0 * TotalIndirectCalls) / TotalCalls) << "%\n"
<< "BOLT-INFO: ICP percentage of indirect calls that can be "
"optimized = "
<< format("%.1f", (100.0 * TotalNumFrequentCalls) /
std::max<size_t>(TotalIndirectCalls, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of indirect callsites that are "
"optimized = "
<< format("%.1f", (100.0 * TotalOptimizedIndirectCallsites) /
std::max<uint64_t>(TotalIndirectCallsites, 1))
<< "%\n"
<< "BOLT-INFO: ICP number of method load elimination candidates = "
<< TotalMethodLoadEliminationCandidates << "\n"
<< "BOLT-INFO: ICP percentage of method calls candidates that have "
"loads eliminated = "
<< format("%.1f", (100.0 * TotalMethodLoadsEliminated) /
std::max<uint64_t>(
TotalMethodLoadEliminationCandidates, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of indirect branches that are "
"optimized = "
<< format("%.1f", (100.0 * TotalNumFrequentJmps) /
std::max<uint64_t>(TotalIndirectJmps, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of jump table callsites that are "
<< "optimized = "
<< format("%.1f", (100.0 * TotalOptimizedJumpTableCallsites) /
std::max<uint64_t>(TotalJumpTableCallsites, 1))
<< "%\n"
<< "BOLT-INFO: ICP number of jump table callsites that can use hot "
<< "indices = " << TotalIndexBasedCandidates << "\n"
<< "BOLT-INFO: ICP percentage of jump table callsites that use hot "
"indices = "
<< format("%.1f", (100.0 * TotalIndexBasedJumps) /
std::max<uint64_t>(TotalIndexBasedCandidates, 1))
<< "%\n";
BC.outs()
<< "BOLT-INFO: ICP total indirect callsites with profile = "
<< TotalIndirectCallsites << "\n"
<< "BOLT-INFO: ICP total jump table callsites = "
<< TotalJumpTableCallsites << "\n"
<< "BOLT-INFO: ICP total number of calls = " << TotalCalls << "\n"
<< "BOLT-INFO: ICP percentage of calls that are indirect = "
<< format("%.1f", (100.0 * TotalIndirectCalls) / TotalCalls) << "%\n"
<< "BOLT-INFO: ICP percentage of indirect calls that can be "
"optimized = "
<< format("%.1f", (100.0 * TotalNumFrequentCalls) /
std::max<size_t>(TotalIndirectCalls, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of indirect callsites that are "
"optimized = "
<< format("%.1f", (100.0 * TotalOptimizedIndirectCallsites) /
std::max<uint64_t>(TotalIndirectCallsites, 1))
<< "%\n"
<< "BOLT-INFO: ICP number of method load elimination candidates = "
<< TotalMethodLoadEliminationCandidates << "\n"
<< "BOLT-INFO: ICP percentage of method calls candidates that have "
"loads eliminated = "
<< format("%.1f",
(100.0 * TotalMethodLoadsEliminated) /
std::max<uint64_t>(TotalMethodLoadEliminationCandidates, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of indirect branches that are "
"optimized = "
<< format("%.1f", (100.0 * TotalNumFrequentJmps) /
std::max<uint64_t>(TotalIndirectJmps, 1))
<< "%\n"
<< "BOLT-INFO: ICP percentage of jump table callsites that are "
<< "optimized = "
<< format("%.1f", (100.0 * TotalOptimizedJumpTableCallsites) /
std::max<uint64_t>(TotalJumpTableCallsites, 1))
<< "%\n"
<< "BOLT-INFO: ICP number of jump table callsites that can use hot "
<< "indices = " << TotalIndexBasedCandidates << "\n"
<< "BOLT-INFO: ICP percentage of jump table callsites that use hot "
"indices = "
<< format("%.1f", (100.0 * TotalIndexBasedJumps) /
std::max<uint64_t>(TotalIndexBasedCandidates, 1))
<< "%\n";

#ifndef NDEBUG
verifyProfile(BFs);
#endif
return Error::success();
}

} // namespace bolt
Expand Down
13 changes: 7 additions & 6 deletions bolt/lib/Passes/Inliner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -496,11 +496,11 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
return DidInlining;
}

void Inliner::runOnFunctions(BinaryContext &BC) {
Error Inliner::runOnFunctions(BinaryContext &BC) {
opts::syncOptions();

if (!opts::inliningEnabled())
return;
return Error::success();

bool InlinedOnce;
unsigned NumIters = 0;
Expand Down Expand Up @@ -540,10 +540,11 @@ void Inliner::runOnFunctions(BinaryContext &BC) {
} while (InlinedOnce && NumIters < opts::InlineMaxIters);

if (NumInlinedCallSites)
outs() << "BOLT-INFO: inlined " << NumInlinedDynamicCalls << " calls at "
<< NumInlinedCallSites << " call sites in " << NumIters
<< " iteration(s). Change in binary size: " << TotalInlinedBytes
<< " bytes.\n";
BC.outs() << "BOLT-INFO: inlined " << NumInlinedDynamicCalls << " calls at "
<< NumInlinedCallSites << " call sites in " << NumIters
<< " iteration(s). Change in binary size: " << TotalInlinedBytes
<< " bytes.\n";
return Error::success();
}

} // namespace bolt
Expand Down
79 changes: 40 additions & 39 deletions bolt/lib/Passes/Instrumentation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ static bool hasAArch64ExclusiveMemop(BinaryFunction &Function) {
for (const MCInst &Inst : BB)
if (BC.MIB->isAArch64Exclusive(Inst)) {
if (opts::Verbosity >= 1)
outs() << "BOLT-INSTRUMENTER: Function " << Function
<< " has exclusive instructions, skip instrumentation\n";
BC.outs() << "BOLT-INSTRUMENTER: Function " << Function
<< " has exclusive instructions, skip instrumentation\n";
return true;
}

Expand Down Expand Up @@ -526,7 +526,7 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
FuncDesc->EdgesSet.clear();
}

void Instrumentation::runOnFunctions(BinaryContext &BC) {
Error Instrumentation::runOnFunctions(BinaryContext &BC) {
const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
Expand Down Expand Up @@ -567,17 +567,16 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {

ErrorOr<BinarySection &> SetupSection =
BC.getUniqueSectionByName("I__setup");
if (!SetupSection) {
llvm::errs() << "Cannot find I__setup section\n";
exit(1);
}
if (!SetupSection)
return createFatalBOLTError("Cannot find I__setup section\n");

MCSymbol *Target = BC.registerNameAtAddress(
"__bolt_instr_setup", SetupSection->getAddress(), 0, 0);
MCInst NewInst;
BC.MIB->createCall(NewInst, Target, BC.Ctx.get());
BB.insertInstruction(BB.begin(), std::move(NewInst));
} else {
llvm::errs() << "BOLT-WARNING: Entry point not found\n";
BC.errs() << "BOLT-WARNING: Entry point not found\n";
}

if (BinaryData *BD = BC.getBinaryDataByName("___GLOBAL_init_65535/1")) {
Expand All @@ -586,10 +585,9 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
BinaryBasicBlock &BB = Ctor->front();
ErrorOr<BinarySection &> FiniSection =
BC.getUniqueSectionByName("I__fini");
if (!FiniSection) {
llvm::errs() << "Cannot find I__fini section\n";
exit(1);
}
if (!FiniSection)
return createFatalBOLTError("Cannot find I__fini section");

MCSymbol *Target = BC.registerNameAtAddress(
"__bolt_instr_fini", FiniSection->getAddress(), 0, 0);
auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); };
Expand All @@ -598,11 +596,12 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
LEA->getOperand(4).setExpr(
MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *BC.Ctx));
} else {
llvm::errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";
BC.errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";
}
}

setupRuntimeLibrary(BC);
return Error::success();
}

void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
Expand Down Expand Up @@ -688,32 +687,34 @@ void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) {
uint32_t FuncDescSize = Summary->getFDSize();

outs() << "BOLT-INSTRUMENTER: Number of indirect call site descriptors: "
<< Summary->IndCallDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of indirect call target descriptors: "
<< Summary->IndCallTargetDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of function descriptors: "
<< Summary->FunctionDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of branch counters: " << BranchCounters
<< "\n";
outs() << "BOLT-INSTRUMENTER: Number of ST leaf node counters: "
<< LeafNodeCounters << "\n";
outs() << "BOLT-INSTRUMENTER: Number of direct call counters: "
<< DirectCallCounters << "\n";
outs() << "BOLT-INSTRUMENTER: Total number of counters: "
<< Summary->Counters.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Total size of counters: "
<< (Summary->Counters.size() * 8) << " bytes (static alloc memory)\n";
outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "
<< Summary->StringTable.size() << " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Total size of descriptors: "
<< (FuncDescSize +
Summary->IndCallDescriptions.size() * sizeof(IndCallDescription) +
Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription))
<< " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "
<< opts::InstrumentationFilename << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of indirect call site descriptors: "
<< Summary->IndCallDescriptions.size() << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of indirect call target descriptors: "
<< Summary->IndCallTargetDescriptions.size() << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of function descriptors: "
<< Summary->FunctionDescriptions.size() << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of branch counters: "
<< BranchCounters << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of ST leaf node counters: "
<< LeafNodeCounters << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Number of direct call counters: "
<< DirectCallCounters << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Total number of counters: "
<< Summary->Counters.size() << "\n";
BC.outs() << "BOLT-INSTRUMENTER: Total size of counters: "
<< (Summary->Counters.size() * 8)
<< " bytes (static alloc memory)\n";
BC.outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "
<< Summary->StringTable.size() << " bytes in file\n";
BC.outs() << "BOLT-INSTRUMENTER: Total size of descriptors: "
<< (FuncDescSize +
Summary->IndCallDescriptions.size() *
sizeof(IndCallDescription) +
Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription))
<< " bytes in file\n";
BC.outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "
<< opts::InstrumentationFilename << "\n";

InstrumentationRuntimeLibrary *RtLibrary =
static_cast<InstrumentationRuntimeLibrary *>(BC.getRuntimeLibrary());
Expand Down
33 changes: 17 additions & 16 deletions bolt/lib/Passes/JTFootprintReduction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ void JTFootprintReduction::optimizeFunction(BinaryFunction &Function,
++I;
}

void JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
Error JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
if (opts::JumpTables == JTS_BASIC && BC.HasRelocations)
return;
return Error::success();

std::unique_ptr<RegAnalysis> RA;
std::unique_ptr<BinaryFunctionCallGraph> CG;
Expand All @@ -272,23 +272,24 @@ void JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
}

if (TotalJTs == TotalJTsDenied) {
outs() << "BOLT-INFO: JT Footprint reduction: no changes were made.\n";
return;
BC.outs() << "BOLT-INFO: JT Footprint reduction: no changes were made.\n";
return Error::success();
}

outs() << "BOLT-INFO: JT Footprint reduction stats (simple funcs only):\n";
BC.outs() << "BOLT-INFO: JT Footprint reduction stats (simple funcs only):\n";
if (OptimizedScore)
outs() << format("\t %.2lf%%", (OptimizedScore * 100.0 / TotalJTScore))
<< " of dynamic JT entries were reduced.\n";
outs() << "\t " << TotalJTs - TotalJTsDenied << " of " << TotalJTs
<< " jump tables affected.\n";
outs() << "\t " << IndJmps - IndJmpsDenied << " of " << IndJmps
<< " indirect jumps to JTs affected.\n";
outs() << "\t " << NumJTsBadMatch
<< " JTs discarded due to unsupported jump pattern.\n";
outs() << "\t " << NumJTsNoReg
<< " JTs discarded due to register unavailability.\n";
outs() << "\t " << BytesSaved << " bytes saved.\n";
BC.outs() << format("\t %.2lf%%", (OptimizedScore * 100.0 / TotalJTScore))
<< " of dynamic JT entries were reduced.\n";
BC.outs() << "\t " << TotalJTs - TotalJTsDenied << " of " << TotalJTs
<< " jump tables affected.\n";
BC.outs() << "\t " << IndJmps - IndJmpsDenied << " of " << IndJmps
<< " indirect jumps to JTs affected.\n";
BC.outs() << "\t " << NumJTsBadMatch
<< " JTs discarded due to unsupported jump pattern.\n";
BC.outs() << "\t " << NumJTsNoReg
<< " JTs discarded due to register unavailability.\n";
BC.outs() << "\t " << BytesSaved << " bytes saved.\n";
return Error::success();
}

} // namespace bolt
Expand Down
Loading