6 changes: 3 additions & 3 deletions bolt/include/bolt/Core/MCPlusBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCDisassembler/MCSymbolizer.h"
#include "llvm/MC/MCExpr.h"
Expand All @@ -27,6 +28,7 @@
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/RWMutex.h"
Expand Down Expand Up @@ -533,9 +535,7 @@ class MCPlusBuilder {
return Analysis->isReturn(Inst);
}

virtual bool isTerminator(const MCInst &Inst) const {
return Analysis->isTerminator(Inst);
}
virtual bool isTerminator(const MCInst &Inst) const;

virtual bool isNoop(const MCInst &Inst) const {
llvm_unreachable("not implemented");
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Passes/BinaryPasses.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
#include "bolt/Core/DynoStats.h"
#include "llvm/Support/CommandLine.h"
#include <atomic>
#include <map>
#include <set>
#include <string>
#include <unordered_set>
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Passes/CacheMetrics.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#ifndef BOLT_PASSES_CACHEMETRICS_H
#define BOLT_PASSES_CACHEMETRICS_H

#include <cstdint>
#include <vector>

namespace llvm {
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Passes/DominatorAnalysis.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

#include "bolt/Passes/DataflowAnalysis.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Timer.h"

namespace opts {
extern llvm::cl::opt<bool> TimeOpts;
Expand Down
2 changes: 0 additions & 2 deletions bolt/include/bolt/Passes/ReachingDefOrUse.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@

#include "bolt/Passes/DataflowAnalysis.h"
#include "bolt/Passes/RegAnalysis.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Timer.h"
#include <optional>

namespace opts {
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Passes/ReachingInsns.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

#include "bolt/Passes/DataflowAnalysis.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Timer.h"

namespace opts {
extern llvm::cl::opt<bool> TimeOpts;
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Passes/ReorderUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#ifndef BOLT_PASSES_REORDER_UTILS_H
#define BOLT_PASSES_REORDER_UTILS_H

#include <memory>
#include <vector>

#include "llvm/ADT/BitVector.h"
Expand Down
27 changes: 19 additions & 8 deletions bolt/include/bolt/Profile/BoltAddressTranslation.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <unordered_map>

namespace llvm {
class MCSymbol;
class raw_ostream;

namespace object {
Expand Down Expand Up @@ -118,10 +119,12 @@ class BoltAddressTranslation {
/// True if a given \p Address is a function with translation table entry.
bool isBATFunction(uint64_t Address) const { return Maps.count(Address); }

/// Returns branch offsets grouped by containing basic block in a given
/// function.
std::unordered_map<uint32_t, std::vector<uint32_t>>
getBFBranches(uint64_t FuncOutputAddress) const;
/// For a given \p Symbol in the output binary and known \p InputOffset
/// return a corresponding pair of parent BinaryFunction and secondary entry
/// point in it.
std::pair<const BinaryFunction *, unsigned>
translateSymbol(const BinaryContext &BC, const MCSymbol &Symbol,
uint32_t InputOffset) const;

private:
/// Helper to update \p Map by inserting one or more BAT entries reflecting
Expand All @@ -146,9 +149,9 @@ class BoltAddressTranslation {
/// entries in function address translation map.
APInt calculateBranchEntriesBitMask(MapTy &Map, size_t EqualElems);

/// Calculate the number of equal offsets (output = input) in the beginning
/// of the function.
size_t getNumEqualOffsets(const MapTy &Map) const;
/// Calculate the number of equal offsets (output = input - skew) in the
/// beginning of the function.
size_t getNumEqualOffsets(const MapTy &Map, uint32_t Skew) const;

std::map<uint64_t, MapTy> Maps;

Expand All @@ -158,6 +161,10 @@ class BoltAddressTranslation {
/// Map a function to its secondary entry points vector
std::unordered_map<uint64_t, std::vector<uint32_t>> SecondaryEntryPointsMap;

/// Return a secondary entry point ID for a function located at \p Address and
/// \p Offset within that function.
unsigned getSecondaryEntryPointId(uint64_t Address, uint32_t Offset) const;

/// Links outlined cold bocks to their original function
std::map<uint64_t, uint64_t> ColdPartSource;

Expand All @@ -181,7 +188,7 @@ class BoltAddressTranslation {
EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {}
};

std::unordered_map<uint32_t, EntryTy> Map;
std::map<uint32_t, EntryTy> Map;
const EntryTy &getEntry(uint32_t BBInputOffset) const {
auto It = Map.find(BBInputOffset);
assert(It != Map.end());
Expand All @@ -206,6 +213,10 @@ class BoltAddressTranslation {
}

size_t getNumBasicBlocks() const { return Map.size(); }

auto begin() const { return Map.begin(); }
auto end() const { return Map.end(); }
auto upper_bound(uint32_t Offset) const { return Map.upper_bound(Offset); }
};

/// Map function output address to its hash and basic blocks hash map.
Expand Down
19 changes: 9 additions & 10 deletions bolt/include/bolt/Profile/DataAggregator.h
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,10 @@ class DataAggregator : public DataReader {
/// Aggregation statistics
uint64_t NumInvalidTraces{0};
uint64_t NumLongRangeTraces{0};
/// Specifies how many samples were recorded in cold areas if we are dealing
/// with profiling data collected in a bolted binary. For LBRs, incremented
/// for the source of the branch to avoid counting cold activity twice (one
/// for source and another for destination).
uint64_t NumColdSamples{0};

/// Looks into system PATH for Linux Perf and set up the aggregator to use it
Expand All @@ -245,14 +249,12 @@ class DataAggregator : public DataReader {
/// disassembled BinaryFunctions
BinaryFunction *getBinaryFunctionContainingAddress(uint64_t Address) const;

/// Perform BAT translation for a given \p Func and return the parent
/// BinaryFunction or nullptr.
BinaryFunction *getBATParentFunction(const BinaryFunction &Func) const;

/// Retrieve the location name to be used for samples recorded in \p Func.
/// If doing BAT translation, link cold parts to the hot part names (used by
/// the original binary). \p Count specifies how many samples were recorded
/// at that location, so we can tally total activity in cold areas if we are
/// dealing with profiling data collected in a bolted binary. For LBRs,
/// \p Count should only be used for the source of the branch to avoid
/// counting cold activity twice (one for source and another for destination).
StringRef getLocationName(BinaryFunction &Func, uint64_t Count);
StringRef getLocationName(const BinaryFunction &Func) const;

/// Semantic actions - parser hooks to interpret parsed perf samples
/// Register a sample (non-LBR mode), i.e. a new hit at \p Address
Expand Down Expand Up @@ -467,9 +469,6 @@ class DataAggregator : public DataReader {
std::error_code writeBATYAML(BinaryContext &BC,
StringRef OutputFilename) const;

/// Fixup profile collected on BOLTed binary, namely handle split functions.
void fixupBATProfile(BinaryContext &BC);

/// Filter out binaries based on PID
void filterBinaryMMapInfo();

Expand Down
2 changes: 1 addition & 1 deletion bolt/include/bolt/Profile/ProfileReaderBase.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class ProfileReaderBase {
/// Return true if the function \p BF may have a profile available.
/// The result is based on the name(s) of the function alone and the profile
/// match is not guaranteed.
virtual bool mayHaveProfileData(const BinaryFunction &BF);
virtual bool mayHaveProfileData(const BinaryFunction &BF) { return true; }

/// Return true if the profile contains an entry for a local object
/// that has an associated file name.
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Profile/ProfileYAMLMapping.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#define BOLT_PROFILE_PROFILEYAMLMAPPING_H

#include "bolt/Core/BinaryFunction.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/YAMLTraits.h"
#include <vector>

Expand Down
13 changes: 11 additions & 2 deletions bolt/include/bolt/Profile/YAMLProfileWriter.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

namespace llvm {
namespace bolt {
class BoltAddressTranslation;
class RewriteInstance;

class YAMLProfileWriter {
Expand All @@ -31,8 +32,16 @@ class YAMLProfileWriter {
/// Save execution profile for that instance.
std::error_code writeProfile(const RewriteInstance &RI);

static yaml::bolt::BinaryFunctionProfile convert(const BinaryFunction &BF,
bool UseDFS);
static yaml::bolt::BinaryFunctionProfile
convert(const BinaryFunction &BF, bool UseDFS,
const BoltAddressTranslation *BAT = nullptr);

/// Set CallSiteInfo destination fields from \p Symbol and return a target
/// BinaryFunction for that symbol.
static const BinaryFunction *
setCSIDestination(const BinaryContext &BC, yaml::bolt::CallSiteInfo &CSI,
const MCSymbol *Symbol, const BoltAddressTranslation *BAT,
uint32_t Offset = 0);
};

} // namespace bolt
Expand Down
2 changes: 0 additions & 2 deletions bolt/include/bolt/Rewrite/DWARFRewriter.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include <vector>

namespace llvm {
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Rewrite/MetadataManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

#include "bolt/Rewrite/MetadataRewriter.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/Error.h"

namespace llvm {
namespace bolt {
Expand Down
11 changes: 0 additions & 11 deletions bolt/include/bolt/Rewrite/RewriteInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#include "bolt/Core/Linker.h"
#include "bolt/Rewrite/MetadataManager.h"
#include "bolt/Utils/NameResolver.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/MC/StringTableBuilder.h"
#include "llvm/Object/ELFObjectFile.h"
#include "llvm/Object/ObjectFile.h"
Expand Down Expand Up @@ -369,13 +368,6 @@ class RewriteInstance {
/// rewritten binary.
void patchBuildID();

/// Return file offset corresponding to a given virtual address.
uint64_t getFileOffsetFor(uint64_t Address) {
assert(Address >= NewTextSegmentAddress &&
"address in not in the new text segment");
return Address - NewTextSegmentAddress + NewTextSegmentOffset;
}

/// Return file offset corresponding to a virtual \p Address.
/// Return 0 if the address has no mapping in the file, including being
/// part of .bss section.
Expand All @@ -399,9 +391,6 @@ class RewriteInstance {
/// Return true if the section holds debug information.
static bool isDebugSection(StringRef SectionName);

/// Return true if the section holds linux kernel symbol information.
static bool isKSymtabSection(StringRef SectionName);

/// Adds Debug section to overwrite.
static void addToDebugSectionsToOverwrite(const char *Section) {
DebugSectionsToOverwrite.emplace_back(Section);
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

#include "bolt/Core/Linker.h"
#include "llvm/ADT/StringRef.h"
#include <functional>
#include <vector>

namespace llvm {
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Utils/NameShortener.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#define BOLT_UTILS_NAME_SHORTENER_H

#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/Twine.h"

namespace llvm {
namespace bolt {
Expand Down
69 changes: 42 additions & 27 deletions bolt/lib/Core/BinaryContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include "bolt/Core/BinaryEmitter.h"
#include "bolt/Core/BinaryFunction.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "bolt/Utils/NameResolver.h"
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Twine.h"
Expand All @@ -39,7 +38,6 @@
#include <algorithm>
#include <functional>
#include <iterator>
#include <numeric>
#include <unordered_set>

using namespace llvm;
Expand Down Expand Up @@ -162,28 +160,30 @@ BinaryContext::~BinaryContext() {

/// Create BinaryContext for a given architecture \p ArchName and
/// triple \p TripleName.
Expected<std::unique_ptr<BinaryContext>>
BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
std::unique_ptr<DWARFContext> DwCtx,
JournalingStreams Logger) {
Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(
Triple TheTriple, StringRef InputFileName, SubtargetFeatures *Features,
bool IsPIC, std::unique_ptr<DWARFContext> DwCtx, JournalingStreams Logger) {
StringRef ArchName = "";
std::string FeaturesStr = "";
switch (File->getArch()) {
switch (TheTriple.getArch()) {
case llvm::Triple::x86_64:
if (Features)
return createFatalBOLTError(
"x86_64 target does not use SubtargetFeatures");
ArchName = "x86-64";
FeaturesStr = "+nopl";
break;
case llvm::Triple::aarch64:
if (Features)
return createFatalBOLTError(
"AArch64 target does not use SubtargetFeatures");
ArchName = "aarch64";
FeaturesStr = "+all";
break;
case llvm::Triple::riscv64: {
ArchName = "riscv64";
Expected<SubtargetFeatures> Features = File->getFeatures();

if (auto E = Features.takeError())
return std::move(E);

if (!Features)
return createFatalBOLTError("RISCV target needs SubtargetFeatures");
// We rely on relaxation for some transformations (e.g., promoting all calls
// to PseudoCALL and then making JITLink relax them). Since the relax
// feature is not stored in the object file, we manually enable it.
Expand All @@ -196,12 +196,11 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
"BOLT-ERROR: Unrecognized machine in ELF file");
}

auto TheTriple = std::make_unique<Triple>(File->makeTriple());
const std::string TripleName = TheTriple->str();
const std::string TripleName = TheTriple.str();

std::string Error;
const Target *TheTarget =
TargetRegistry::lookupTarget(std::string(ArchName), *TheTriple, Error);
TargetRegistry::lookupTarget(std::string(ArchName), TheTriple, Error);
if (!TheTarget)
return createStringError(make_error_code(std::errc::not_supported),
Twine("BOLT-ERROR: ", Error));
Expand Down Expand Up @@ -240,13 +239,13 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
Twine("BOLT-ERROR: no instruction info for target ", TripleName));

std::unique_ptr<MCContext> Ctx(
new MCContext(*TheTriple, AsmInfo.get(), MRI.get(), STI.get()));
new MCContext(TheTriple, AsmInfo.get(), MRI.get(), STI.get()));
std::unique_ptr<MCObjectFileInfo> MOFI(
TheTarget->createMCObjectFileInfo(*Ctx, IsPIC));
Ctx->setObjectFileInfo(MOFI.get());
// We do not support X86 Large code model. Change this in the future.
bool Large = false;
if (TheTriple->getArch() == llvm::Triple::aarch64)
if (TheTriple.getArch() == llvm::Triple::aarch64)
Large = true;
unsigned LSDAEncoding =
Large ? dwarf::DW_EH_PE_absptr : dwarf::DW_EH_PE_udata4;
Expand All @@ -273,7 +272,7 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,

int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
std::unique_ptr<MCInstPrinter> InstructionPrinter(
TheTarget->createMCInstPrinter(*TheTriple, AsmPrinterVariant, *AsmInfo,
TheTarget->createMCInstPrinter(TheTriple, AsmPrinterVariant, *AsmInfo,
*MII, *MRI));
if (!InstructionPrinter)
return createStringError(
Expand All @@ -285,8 +284,8 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
TheTarget->createMCCodeEmitter(*MII, *Ctx));

auto BC = std::make_unique<BinaryContext>(
std::move(Ctx), std::move(DwCtx), std::move(TheTriple), TheTarget,
std::string(TripleName), std::move(MCE), std::move(MOFI),
std::move(Ctx), std::move(DwCtx), std::make_unique<Triple>(TheTriple),
TheTarget, std::string(TripleName), std::move(MCE), std::move(MOFI),
std::move(AsmInfo), std::move(MII), std::move(STI),
std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI),
std::move(DisAsm), Logger);
Expand All @@ -296,7 +295,7 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
BC->MAB = std::unique_ptr<MCAsmBackend>(
BC->TheTarget->createMCAsmBackend(*BC->STI, *BC->MRI, MCTargetOptions()));

BC->setFilename(File->getFileName());
BC->setFilename(InputFileName);

BC->HasFixedLoadAddress = !IsPIC;

Expand Down Expand Up @@ -556,6 +555,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
const uint64_t NextJTAddress,
JumpTable::AddressesType *EntriesAsAddress,
bool *HasEntryInFragment) const {
// Target address of __builtin_unreachable.
const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();

// Is one of the targets __builtin_unreachable?
bool HasUnreachable = false;

Expand All @@ -565,9 +567,15 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
// Number of targets other than __builtin_unreachable.
uint64_t NumRealEntries = 0;

auto addEntryAddress = [&](uint64_t EntryAddress) {
if (EntriesAsAddress)
EntriesAsAddress->emplace_back(EntryAddress);
// Size of the jump table without trailing __builtin_unreachable entries.
size_t TrimmedSize = 0;

auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {
if (!EntriesAsAddress)
return;
EntriesAsAddress->emplace_back(EntryAddress);
if (!Unreachable)
TrimmedSize = EntriesAsAddress->size();
};

ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
Expand Down Expand Up @@ -619,8 +627,8 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
: *getPointerAtAddress(EntryAddress);

// __builtin_unreachable() case.
if (Value == BF.getAddress() + BF.getSize()) {
addEntryAddress(Value);
if (Value == UnreachableAddress) {
addEntryAddress(Value, /*Unreachable*/ true);
HasUnreachable = true;
LLVM_DEBUG(dbgs() << formatv("OK: {0:x} __builtin_unreachable\n", Value));
continue;
Expand Down Expand Up @@ -674,6 +682,13 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
addEntryAddress(Value);
}

// Trim direct/normal jump table to exclude trailing unreachable entries that
// can collide with a function address.
if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&
TrimmedSize != EntriesAsAddress->size() &&
getBinaryFunctionAtAddress(UnreachableAddress))
EntriesAsAddress->resize(TrimmedSize);

// It's a jump table if the number of real entries is more than 1, or there's
// one real entry and one or more special targets. If there are only multiple
// special targets, then it's not a jump table.
Expand Down Expand Up @@ -1865,7 +1880,7 @@ MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {
// For aarch64 and riscv, the ABI defines mapping symbols so we identify data
// in the code section (see IHI0056B). $x identifies a symbol starting code or
// the end of a data chunk inside code, $d identifies start of data.
if ((!isAArch64() && !isRISCV()) || ELFSymbolRef(Symbol).getSize())
if (isX86() || ELFSymbolRef(Symbol).getSize())
return MarkerSymType::NONE;

Expected<StringRef> NameOrError = Symbol.getName();
Expand Down
Loading