20 changes: 20 additions & 0 deletions compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@
#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON)
#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON)
#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON)
#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON)
#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON)
#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON)
#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON)
#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON)
#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON)
#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON)
#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON)
#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON)
Expand All @@ -41,6 +45,10 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
COMPILER_RT_WEAK;
extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
Expand All @@ -63,6 +71,18 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) {
return &PROF_VNAME_START;
}
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) {
return &PROF_VNAME_STOP;
}
COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) {
return &PROF_VTABLE_START;
}
COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_end_vtables(void) {
return &PROF_VTABLE_STOP;
}
COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) {
return &PROF_CNTS_START;
}
Expand Down
37 changes: 29 additions & 8 deletions compiler-rt/lib/profile/InstrProfilingWriter.c
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,14 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
const char *BitmapEnd = __llvm_profile_end_bitmap();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
const char *VNamesBegin = __llvm_profile_begin_vtabnames();
const char *VNamesEnd = __llvm_profile_end_vtabnames();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
NamesBegin, NamesEnd, SkipNameDataWrite);
NamesBegin, NamesEnd, VTableBegin, VTableEnd,
VNamesBegin, VNamesEnd, SkipNameDataWrite);
}

COMPILER_RT_VISIBILITY int
Expand All @@ -261,7 +266,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const char *CountersBegin, const char *CountersEnd,
const char *BitmapBegin, const char *BitmapEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
const char *NamesEnd, int SkipNameDataWrite) {
const char *NamesEnd, const VTableProfData *VTableBegin,
const VTableProfData *VTableEnd, const char *VNamesBegin,
const char *VNamesEnd, int SkipNameDataWrite) {
/* Calculate size of sections. */
const uint64_t DataSectionSize =
__llvm_profile_get_data_size(DataBegin, DataEnd);
Expand All @@ -273,18 +280,28 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const uint64_t NumBitmapBytes =
__llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);
const uint64_t NumVTables =
__llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
const uint64_t VTableSectionSize =
__llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
const uint64_t VNamesSize =
__llvm_profile_get_name_size(VNamesBegin, VNamesEnd);

/* Create the header. */
__llvm_profile_header Header;

/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
__llvm_profile_get_padding_sizes_for_counters(
DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
&PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
&PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames);
PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
PaddingBytesAfterVTable, PaddingBytesAfterVNames;
if (__llvm_profile_get_padding_sizes_for_counters(
DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,
&PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
&PaddingBytesAfterNames, &PaddingBytesAfterVTable,
&PaddingBytesAfterVNames) == -1)
return -1;

{
/* Initialize header structure. */
Expand Down Expand Up @@ -323,7 +340,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
{BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
{SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
{NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
{VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
{SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;

Expand Down
61 changes: 56 additions & 5 deletions llvm/include/llvm/Analysis/IndirectCallVisitor.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,27 +12,78 @@
#ifndef LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
#define LLVM_ANALYSIS_INDIRECTCALLVISITOR_H

#include "llvm/ADT/SetVector.h"
#include "llvm/IR/InstVisitor.h"
#include <vector>

namespace llvm {
// Visitor class that finds all indirect call.
// Visitor class that finds indirect calls or instructions that gives vtable
// value, depending on Type.
struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
enum class InstructionType {
kIndirectCall = 0,
kVTableVal = 1,
};
std::vector<CallBase *> IndirectCalls;
PGOIndirectCallVisitor() = default;
std::vector<Instruction *> ProfiledAddresses;
PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}

void visitCallBase(CallBase &Call) {
if (Call.isIndirectCall())
if (Call.isIndirectCall()) {
IndirectCalls.push_back(&Call);

if (Type != InstructionType::kVTableVal)
return;

LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
// The code pattern to look for
//
// %vtable = load ptr, ptr %b
// %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
// %2 = load ptr, ptr %vfn
// %call = tail call i32 %2(ptr %b)
//
// %vtable is the vtable address value to profile, and
// %2 is the indirect call target address to profile.
if (LI != nullptr) {
Value *Ptr = LI->getPointerOperand();
Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
// This is a heuristic to find address feeding instructions.
// FIXME: Add support in the frontend so LLVM type intrinsics are
// emitted without LTO. This way, added intrinsics could filter
// non-vtable instructions and reduce instrumentation overhead.
// Since a non-vtable profiled address is not within the address
// range of vtable objects, it's stored as zero in indexed profiles.
// A pass that looks up symbol with an zero hash will (almost) always
// find nullptr and skip the actual transformation (e.g., comparison
// of symbols). So the performance overhead from non-vtable profiled
// address is negligible if exists at all. Comparing loaded address
// with symbol address guarantees correctness.
if (VTablePtr != nullptr && isa<Instruction>(VTablePtr)) {
ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
}
}
}
}

private:
InstructionType Type;
};

// Helper function that finds all indirect call sites.
inline std::vector<CallBase *> findIndirectCalls(Function &F) {
PGOIndirectCallVisitor ICV;
PGOIndirectCallVisitor ICV(
PGOIndirectCallVisitor::InstructionType::kIndirectCall);
ICV.visit(F);
return ICV.IndirectCalls;
}

inline std::vector<Instruction *> findVTableAddrs(Function &F) {
PGOIndirectCallVisitor ICV(
PGOIndirectCallVisitor::InstructionType::kVTableVal);
ICV.visit(F);
return ICV.ProfiledAddresses;
}

} // namespace llvm

#endif
156 changes: 141 additions & 15 deletions llvm/include/llvm/ProfileData/InstrProf.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() {
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }

/// Return the name prefix of variables containing virtual table profile data.
inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }

/// Return the name prefix of variables containing per-function control data.
inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }

Expand All @@ -110,6 +113,8 @@ inline StringRef getInstrProfNamesVarName() {
return "__llvm_prf_nm";
}

inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }

/// Return the name of a covarage mapping variable (internal linkage)
/// for each instrumented source module. Such variables are allocated
/// in the __llvm_covmap section.
Expand Down Expand Up @@ -246,6 +251,9 @@ Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
std::string &Result, bool doCompression = true);

Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression);

/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
Expand Down Expand Up @@ -288,14 +296,16 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
/// Return the PGOFuncName meta data associated with a function.
MDNode *getPGOFuncNameMetadata(const Function &F);

std::string getPGOName(const GlobalVariable &V, bool InLTO = false);

/// Create the PGOFuncName meta data if PGOFuncName is different from
/// function's raw name. This should only apply to internal linkage functions
/// declared by users only.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);

/// Check if we can use Comdat for profile variables. This will eliminate
/// the duplicated profile variables for Comdat functions.
bool needsComdatForCounter(const Function &F, const Module &M);
bool needsComdatForCounter(const GlobalValue &GV, const Module &M);

/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind {
Expand Down Expand Up @@ -429,20 +439,36 @@ uint64_t ComputeHash(StringRef K);
class InstrProfSymtab {
public:
using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
using RangeHashMap =
std::vector<std::pair<std::pair<uint64_t, uint64_t>, uint64_t>>;

private:
StringRef Data;
uint64_t Address = 0;
// Unique name strings.
// Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
// going to be sorted) has unique MD5 keys in the first place.
StringSet<> NameTab;
// Records the unique virtual table names. This is used by InstrProfWriter to
// write out an on-disk chained hash table of virtual table names.
// InstrProfWriter stores per function profile data (keyed by function names)
// so it doesn't use a StringSet for function names.
StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
// A map from MD5 keys to virtual table definitions. Only populated when
// building the Symtab from a module.
std::vector<std::pair<uint64_t, GlobalVariable *>> MD5VTableMap;
// A map from MD5 keys to function define. We only populate this map
// when build the Symtab from a Module.
std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
// A map from function runtime address to function name MD5 hash.
// This map is only populated and used by raw instr profile reader.
AddrHashMap AddrToMD5Map;
// A map from virtual table runtime address to function name MD5 hash.
// This map is only populated and used by raw instr profile reader.
// This is a different map from 'AddrToMD5Map' for readability and
// debuggability.
RangeHashMap VTableAddrRangeToMD5Map;
bool Sorted = false;

static StringRef getExternalSymbol() {
Expand Down Expand Up @@ -470,9 +496,19 @@ class InstrProfSymtab {

/// \c NameStrings is a string composed of one of more sub-strings
/// encoded in the format described in \c collectPGOFuncNameStrings.
/// This method is a wrapper to \c readPGOFuncNameStrings method.
/// This method is a wrapper to \c readAndDecodeStrings method.
Error create(StringRef NameStrings);

/// \c FuncNameStrings is a string composed of one or more encoded function
/// name strings, and \c VTableNameStrings composes of one or more encoded
/// vtable names. This function is a wrapper to \c readAndDecodeStrings
/// method.
Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);

/// Initialize 'this' with the set of vtable names encoded in
/// \c CompressedVTableNames.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);

/// This interface is used by reader of CoverageMapping test
/// format.
inline Error create(StringRef D, uint64_t BaseAddr);
Expand All @@ -485,32 +521,70 @@ class InstrProfSymtab {

/// Create InstrProfSymtab from a set of names iteratable from
/// \p IterRange. This interface is used by IndexedProfReader.
template <typename NameIterRange> Error create(const NameIterRange &IterRange);

/// Update the symtab by adding \p FuncName to the table. This interface
/// is used by the raw and text profile readers.
Error addFuncName(StringRef FuncName) {
if (FuncName.empty())
template <typename NameIterRange>
Error create(const NameIterRange &IterRange);

/// Create InstrProfSymtab from a set of function names and vtable
/// names iteratable from \p IterRange. This interface is used by
/// IndexedProfReader.
template <typename FuncNameIterRange, typename VTableNameIterRange>
Error create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange);

Error addSymbolName(StringRef SymbolName) {
if (SymbolName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
"function name is empty");
auto Ins = NameTab.insert(FuncName);
"symbol name is empty");

// Insert into NameTab so that MD5NameMap (a vector that will be sorted)
// won't have duplicated entries in the first place.
auto Ins = NameTab.insert(SymbolName);
if (Ins.second) {
MD5NameMap.push_back(std::make_pair(
IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey()));
Sorted = false;
}
return Error::success();
}

/// The method name is kept since there are many callers.
/// It just forwards to 'addSymbolName'.
Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }

/// Adds VTableName as a known symbol, and inserts it to a map that
/// tracks all vtable names.
Error addVTableName(StringRef VTableName) {
if (Error E = addSymbolName(VTableName))
return E;

// Record VTableName. InstrProfWriter uses this map. The comment around
// class member explains why.
VTableNames.insert(VTableName);
return Error::success();
}

const StringSet<> &getVTableNames() const { return VTableNames; }

/// Map a function address to its name's MD5 hash. This interface
/// is only used by the raw profiler reader.
void mapAddress(uint64_t Addr, uint64_t MD5Val) {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}

/// Map the address range (i.e., [start_address, end_address]) of a variable
/// to its names' MD5 hash. This interface is only used by the raw profile
/// reader.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
VTableAddrRangeToMD5Map.push_back(
std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val));
}

/// Return a function's hash, or 0, if the function isn't in this SymTab.
uint64_t getFunctionHashFromAddress(uint64_t Address);

/// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
uint64_t getVTableHashFromAddress(uint64_t Address);

/// Return function's PGO name from the function name's symbol
/// address in the object file. If an error occurs, return
/// an empty string.
Expand All @@ -532,6 +606,8 @@ class InstrProfSymtab {

/// Return function from the name's md5 hash. Return nullptr if not found.
inline Function *getFunction(uint64_t FuncMD5Hash);
// Return vtable from the name's MD5 hash. Return nullptr if not found.
inline GlobalVariable *getGlobalVariable(uint64_t GlobalVariableMD5Hash);

/// Return the name section data.
inline StringRef getNameData() const { return Data; }
Expand All @@ -556,6 +632,23 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) {
return Error::success();
}

template <typename FuncNameIterRange, typename VTableNameIterRange>
Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange) {
for (auto Name : FuncIterRange)
if (Error E = addFuncName(Name))
return E;

for (auto VTableName : VTableIterRange) {
if (Error E = addVTableName(VTableName)) {
return E;
}
}

finalizeSymtab();
return Error::success();
}

void InstrProfSymtab::finalizeSymtab() {
if (Sorted)
return;
Expand All @@ -564,6 +657,13 @@ void InstrProfSymtab::finalizeSymtab() {
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
AddrToMD5Map.end());
// VTable object address ranges should not overlap; so sort by either
// beginning address or end address is fine.
llvm::sort(VTableAddrRangeToMD5Map, less_first());
// std::unique uses == operator for std::pair.
VTableAddrRangeToMD5Map.erase(std::unique(VTableAddrRangeToMD5Map.begin(),
VTableAddrRangeToMD5Map.end()),
VTableAddrRangeToMD5Map.end());
Sorted = true;
}

Expand Down Expand Up @@ -594,6 +694,19 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
return nullptr;
}

GlobalVariable *
InstrProfSymtab::getGlobalVariable(uint64_t GlobalVariableMD5Hash) {
finalizeSymtab();
auto Result =
llvm::lower_bound(MD5VTableMap, GlobalVariableMD5Hash,
[](const std::pair<uint64_t, GlobalVariable *> &LHS,
uint64_t RHS) { return LHS.first < RHS; });

if (Result != MD5VTableMap.end() && Result->first == GlobalVariableMD5Hash)
return Result->second;
return nullptr;
}

// To store the sums of profile count values, or the percentage of
// the sums of the total count values.
struct CountSumOrPercent {
Expand Down Expand Up @@ -820,6 +933,7 @@ struct InstrProfRecord {
struct ValueProfData {
std::vector<InstrProfValueSiteRecord> IndirectCallSites;
std::vector<InstrProfValueSiteRecord> MemOPSizes;
std::vector<InstrProfValueSiteRecord> VTableTargets;
};
std::unique_ptr<ValueProfData> ValueData;

Expand All @@ -842,6 +956,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
case IPVK_VTableTarget:
return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
Expand All @@ -856,6 +972,8 @@ struct InstrProfRecord {
return ValueData->IndirectCallSites;
case IPVK_MemOPSize:
return ValueData->MemOPSizes;
case IPVK_VTableTarget:
return ValueData->VTableTargets;
default:
llvm_unreachable("Unknown value kind!");
}
Expand Down Expand Up @@ -1025,7 +1143,9 @@ enum ProfVersion {
Version10 = 10,
// An additional field is used for bitmap bytes.
Version11 = 11,
// The current version is 11.
// VTable profiling,
Version12 = 12,
// The current version is 12.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
const uint64_t Version = ProfVersion::CurrentVersion;
Expand All @@ -1046,6 +1166,7 @@ struct Header {
uint64_t MemProfOffset;
uint64_t BinaryIdOffset;
uint64_t TemporalProfTracesOffset;
uint64_t VTableNamesOffset; // Organize virtual table names.
// New fields should only be added at the end to ensure that the size
// computation is correct. The methods below need to be updated to ensure that
// the new field is read correctly.
Expand Down Expand Up @@ -1182,8 +1303,13 @@ template <> inline uint64_t getMagic<uint32_t>() {
// It should also match the synthesized type in
// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
template <class IntPtrT> struct alignas(8) ProfileData {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};

template <class IntPtrT> struct alignas(8) VTableProfileData {
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};

// File header structure of the LLVM profile data in raw format.
Expand Down
40 changes: 33 additions & 7 deletions llvm/include/llvm/ProfileData/InstrProfData.inc
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,22 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */

#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(
const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash,
ConstantInt::get(llvm::Type::getInt64Ty(Ctx),
IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx),
VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize,
ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
VTableSizeVal))
#undef INSTR_PROF_VTABLE_DATA
/* INSTR_PROF_VTABLE_DATA end. */

/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
Expand Down Expand Up @@ -145,6 +161,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
(uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
Expand Down Expand Up @@ -186,13 +204,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")

#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
Expand Down Expand Up @@ -267,7 +286,6 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \
#undef COVMAP_HEADER
/* COVMAP_HEADER end. */


#ifdef INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_DATA_DEFINED
INSTR_PROF_SECT_ENTRY(IPSK_data, \
Expand All @@ -282,12 +300,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vname, \
INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
Expand All @@ -307,7 +331,6 @@ INSTR_PROF_SECT_ENTRY(IPSK_covname, \
#undef INSTR_PROF_SECT_ENTRY
#endif


#ifdef INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_DATA_DEFINED

Expand Down Expand Up @@ -347,7 +370,7 @@ typedef struct ValueProfRecord {
/*!
* Return the number of value sites.
*/
uint32_t getNumValueSites() const { return NumValueSites; }
uint32_t getNumValueSites() const { return NumValueSites; }
/*!
* Read data from this record and save it to Record.
*/
Expand Down Expand Up @@ -479,7 +502,6 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites);
#undef INSTR_PROF_VALUE_PROF_DATA
#endif /* INSTR_PROF_VALUE_PROF_DATA */


#ifdef INSTR_PROF_COMMON_API_IMPL
#define INSTR_PROF_DATA_DEFINED
#ifdef __cplusplus
Expand Down Expand Up @@ -663,9 +685,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129

/* Raw profile format version (start from 1). */
#define INSTR_PROF_RAW_VERSION 9
#define INSTR_PROF_RAW_VERSION 10
/* Indexed profile format version (start from 1). */
#define INSTR_PROF_INDEX_VERSION 11
#define INSTR_PROF_INDEX_VERSION 12
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 6

Expand Down Expand Up @@ -703,10 +725,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
Expand All @@ -717,10 +741,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_BITS_COFF ".lprfb$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
/* Since cov data and cov names sections are not allocated, we don't need to
Expand Down
20 changes: 20 additions & 0 deletions llvm/include/llvm/ProfileData/InstrProfReader.h
Original file line number Diff line number Diff line change
Expand Up @@ -326,12 +326,16 @@ class RawInstrProfReader : public InstrProfReader {
uint64_t NamesDelta;
const RawInstrProf::ProfileData<IntPtrT> *Data;
const RawInstrProf::ProfileData<IntPtrT> *DataEnd;
const RawInstrProf::VTableProfileData<IntPtrT> *VTableBegin = nullptr;
const RawInstrProf::VTableProfileData<IntPtrT> *VTableEnd = nullptr;
const char *CountersStart;
const char *CountersEnd;
const char *BitmapStart;
const char *BitmapEnd;
const char *NamesStart;
const char *NamesEnd;
const char *VNamesStart = nullptr;
const char *VNamesEnd = nullptr;
// After value profile is all read, this pointer points to
// the header of next profile data (if exists)
const uint8_t *ValueDataStart;
Expand Down Expand Up @@ -622,6 +626,12 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
InstrProfKind getProfileKind() const override;

Error populateSymtab(InstrProfSymtab &Symtab) override {
// FIXME: the create method calls 'finalizeSymtab' and sorts a bunch of
// arrays/maps. Since there are other data sources other than 'HashTable' to
// populate a symtab, it might make sense to have something like this
// 1. Let each data source populate Symtab and init the arrays/maps without
// calling 'finalizeSymtab'
// 2. Call 'finalizeSymtab' once to get all arrays/maps sorted if needed.
return Symtab.create(HashTable->keys());
}
};
Expand Down Expand Up @@ -656,6 +666,16 @@ class IndexedInstrProfReader : public InstrProfReader {
std::unique_ptr<MemProfRecordHashTable> MemProfRecordTable;
/// MemProf frame profile data on-disk indexed via frame id.
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
/// The reader itself doesn't decompress vtable names. A compiler that reads
/// indexed profiles could construct symtab from module IR so it doesn't need
/// the decompressed names.
/// When a symtab is constructed from profiles by llvm-profdata, the list of
/// names could be decompressed based on `VTableNamePtr` and
/// `CompressedVTableNamesLen`.
/// VTableNamePtr points to the beginning of compressed vtable names.
const char *VTableNamePtr = nullptr;
/// The length of compressed vtable names.
uint64_t CompressedVTableNamesLen = 0;
/// Total size of binary ids.
uint64_t BinaryIdsSize{0};
/// Start address of binary id length and data pairs.
Expand Down
4 changes: 4 additions & 0 deletions llvm/include/llvm/ProfileData/InstrProfWriter.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ class InstrProfWriter {
// List of binary ids.
std::vector<llvm::object::BuildID> BinaryIds;

// Read the vtable names from raw instr profile reader.
StringSet<> VTableNames;

// An enum describing the attributes of the profile.
InstrProfKind ProfileKind = InstrProfKind::Unknown;
// Use raw pointer here for the incomplete type object.
Expand All @@ -84,6 +87,7 @@ class InstrProfWriter {
void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) {
addRecord(std::move(I), 1, Warn);
}
void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); }

/// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the
/// total number of temporal profiling traces the source has seen.
Expand Down
100 changes: 94 additions & 6 deletions llvm/lib/ProfileData/InstrProf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,12 @@ cl::opt<bool> DoInstrProfNameCompression(
"enable-name-compression",
cl::desc("Enable name/filename string compression"), cl::init(true));

cl::opt<bool> EnableVTableValueProfiling(
"enable-vtable-value-profiling", cl::init(false),
cl::desc("If true, the virtual table address will be instrumented to know "
"the types of a C++ pointer. The information is used in indirect "
"call promotion to do selective vtable-based comparison."));

std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
bool AddSegmentInfo) {
Expand Down Expand Up @@ -378,6 +384,13 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
}

std::string getPGOName(const GlobalVariable &V, bool InLTO) {
// PGONameMetadata should be set by compiler at profile use time
// and read by symtab creation to look up symbols corresponding to
// a MD5 hash.
return getIRPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */);
}

// See getIRPGOFuncName() for a discription of the format.
std::pair<StringRef, StringRef>
getParsedIRPGOFuncName(StringRef IRPGOFuncName) {
Expand Down Expand Up @@ -460,6 +473,17 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
return E;
}

SmallVector<MDNode *, 2> Types;
for (GlobalVariable &G : M.globals()) {
if (!G.hasName())
continue;
Types.clear();
G.getMetadata(LLVMContext::MD_type, Types);
if (!Types.empty()) {
MD5VTableMap.emplace_back(G.getGUID(), &G);
}
}
Sorted = false;
finalizeSymtab();
return Error::success();
Expand Down Expand Up @@ -518,6 +542,25 @@ Error InstrProfSymtab::create(StringRef NameStrings) {
std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
}

Error InstrProfSymtab::create(StringRef FuncNameStrings,
StringRef VTableNameStrings) {
if (Error E = readAndDecodeStrings(FuncNameStrings,
std::bind(&InstrProfSymtab::addFuncName,
this, std::placeholders::_1)))
return E;

return readAndDecodeStrings(
VTableNameStrings,
std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
}

Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
StringRef CompressedVTableStrings) {
return readAndDecodeStrings(
CompressedVTableStrings,
std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
}

Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
if (Error E = addFuncName(PGOFuncName))
return E;
Expand Down Expand Up @@ -550,6 +593,28 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
return Error::success();
}

uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
finalizeSymtab();
auto It = lower_bound(
VTableAddrRangeToMD5Map, Address,
[](std::pair<std::pair<uint64_t, uint64_t>, uint64_t> VTableRangeAddr,
uint64_t Addr) {
// Find the first address range of which end address is larger than
// `Addr`. Smaller-than-or-equal-to is used because the profiled address
// within a vtable should be [start-address, end-address).
return VTableRangeAddr.first.second <= Addr;
});

// Returns the MD5 hash if Address is within the address range of an entry.
if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) {
return It->second;
}
// The virtual table address collected from value profiler could be defined
// in another module that is not instrumented. Force the value to be 0 in
// this case.
return 0;
}

uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
finalizeSymtab();
auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
Expand Down Expand Up @@ -626,6 +691,17 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
NameStrs, compression::zlib::isAvailable() && doCompression, Result);
}

Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression) {
std::vector<std::string> VTableNameStrs;
for (auto *VTable : VTables) {
VTableNameStrs.push_back(getPGOName(*VTable));
}
return collectGlobalObjectNameStrings(
VTableNameStrs, compression::zlib::isAvailable() && doCompression,
Result);
}

void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
Expand Down Expand Up @@ -888,6 +964,9 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
if (ValueKind == IPVK_IndirectCallTarget)
return SymTab->getFunctionHashFromAddress(Value);

if (ValueKind == IPVK_VTableTarget)
return SymTab->getVTableHashFromAddress(Value);

return Value;
}

Expand Down Expand Up @@ -1278,8 +1357,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
F.setMetadata(getPGOFuncNameMetadataName(), N);
}

bool needsComdatForCounter(const Function &F, const Module &M) {
if (F.hasComdat())
bool needsComdatForCounter(const GlobalValue &GV, const Module &M) {
if (GV.hasComdat())
return true;

if (!Triple(M.getTargetTriple()).supportsCOMDAT())
Expand All @@ -1295,7 +1374,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) {
// available_externally functions will end up being duplicated in raw profile
// data. This can result in distorted profile as the counts of those dups
// will be accumulated by the profile merger.
GlobalValue::LinkageTypes Linkage = F.getLinkage();
GlobalValue::LinkageTypes Linkage = GV.getLinkage();
if (Linkage != GlobalValue::ExternalWeakLinkage &&
Linkage != GlobalValue::AvailableExternallyLinkage)
return false;
Expand Down Expand Up @@ -1451,14 +1530,17 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
continue;
char ProfileKindName[20];
char ProfileKindName[20] = {0};
switch (I) {
case IPVK_IndirectCallTarget:
strncpy(ProfileKindName, "IndirectCall", 19);
break;
case IPVK_MemOPSize:
strncpy(ProfileKindName, "MemOP", 19);
break;
case IPVK_VTableTarget:
strncpy(ProfileKindName, "VTable", 19);
break;
default:
snprintf(ProfileKindName, 19, "VP[%d]", I);
break;
Expand Down Expand Up @@ -1523,9 +1605,12 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
// When a new field is added in the header add a case statement here to
// populate it.
static_assert(
IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
"Please update the reading code below if a new field has been added, "
"if not add a case statement to fall through to the latest version.");
case 12ull:
H.VTableNamesOffset = read(Buffer, offsetOf(&Header::VTableNamesOffset));
[[fallthrough]];
case 11ull:
[[fallthrough]];
case 10ull:
Expand All @@ -1551,10 +1636,13 @@ size_t Header::size() const {
// When a new field is added to the header add a case statement here to
// compute the size as offset of the new field + size of the new field. This
// relies on the field being added to the end of the list.
static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
"Please update the size computation below if a new field has "
"been added to the header, if not add a case statement to "
"fall through to the latest version.");
case 12ull:
return offsetOf(&Header::VTableNamesOffset) +
sizeof(Header::VTableNamesOffset);
case 11ull:
[[fallthrough]];
case 10ull:
Expand Down
72 changes: 68 additions & 4 deletions llvm/lib/ProfileData/InstrProfReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,14 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
return E;
Value = IndexedInstrProf::ComputeHash(VD.first);
}
} else if (ValueKind == IPVK_VTableTarget) {
if (InstrProfSymtab::isExternalSymbol(VD.first)) {
Value = 0;
} else {
if (Error E = Symtab->addVTableName(VD.first))
return E;
Value = IndexedInstrProf::ComputeHash(VD.first);
}
} else {
READ_NUM(VD.first, Value);
}
Expand Down Expand Up @@ -533,14 +541,30 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {

template <class IntPtrT>
Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),
StringRef(VNamesStart, VNamesEnd - VNamesStart)))
return error(std::move(E));
for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
const IntPtrT FPtr = swap(I->FunctionPointer);
if (!FPtr)
continue;
Symtab.mapAddress(FPtr, swap(I->NameRef));
}

if (VTableBegin != nullptr && VTableEnd != nullptr) {
for (const RawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;
I != VTableEnd; ++I) {
const IntPtrT VPtr = swap(I->VTablePointer);
if (!VPtr)
continue;
// Map both begin and end address to the name hash, since the instrumented
// address could be somewhere in the middle.
// VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks
// the end of vtable address.
Symtab.mapVTableAddress(VPtr, VPtr + swap(I->VTableSize),
swap(I->VTableNameHash));
}
}
return success();
}

Expand Down Expand Up @@ -582,10 +606,17 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
auto NumBitmapBytes = swap(Header.NumBitmapBytes);
auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes);
auto NamesSize = swap(Header.NamesSize);
auto VTableNameSize = swap(Header.VNamesSize);
auto NumVTables = swap(Header.NumVTables);
ValueKindLast = swap(Header.ValueKindLast);

auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
auto PaddingSize = getNumPaddingBytes(NamesSize);
auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);

auto VTableSectionSize =
NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);

// Profile data starts after profile header and binary ids if exist.
ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize;
Expand All @@ -594,7 +625,12 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
CountersOffset + CountersSize + PaddingBytesAfterCounters;
ptrdiff_t NamesOffset =
BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
ptrdiff_t VTableProfDataOffset =
NamesOffset + NamesSize + PaddingBytesAfterNames;
ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
PaddingBytesAfterVTableProfData;
ptrdiff_t ValueDataOffset =
VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;

auto *Start = reinterpret_cast<const char *>(&Header);
if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
Expand All @@ -614,8 +650,14 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
Start + DataOffset);
DataEnd = Data + NumData;
VTableBegin =
reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
Start + VTableProfDataOffset);
VTableEnd = VTableBegin + NumVTables;
NamesStart = Start + NamesOffset;
NamesEnd = NamesStart + NamesSize;
VNamesStart = Start + VTableNameOffset;
VNamesEnd = VNamesStart + VTableNameSize;
}

CountersStart = Start + CountersOffset;
Expand Down Expand Up @@ -1260,6 +1302,19 @@ Error IndexedInstrProfReader::readHeader() {
"corrupted binary ids");
}

if (GET_VERSION(Header->formatVersion()) >= 12) {
uint64_t VTableNamesOffset =
endian::byte_swap<uint64_t, llvm::endianness::little>(
Header->VTableNamesOffset);
const unsigned char *Ptr = Start + VTableNamesOffset;

CompressedVTableNamesLen =
support::endian::readNext<uint64_t, llvm::endianness::little,
unaligned>(Ptr);

VTableNamePtr = (const char *)Ptr;
}

if (GET_VERSION(Header->formatVersion()) >= 10 &&
Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
uint64_t TemporalProfTracesOffset =
Expand Down Expand Up @@ -1319,7 +1374,16 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
if (Symtab)
return *Symtab;

std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
std::unique_ptr<InstrProfSymtab> NewSymtab =
std::make_unique<InstrProfSymtab>();

if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
}

// finalizeSymtab is called inside populateSymtab.
if (Error E = Index->populateSymtab(*NewSymtab)) {
auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
consumeError(error(ErrCode, Msg));
Expand Down
59 changes: 54 additions & 5 deletions llvm/lib/ProfileData/InstrProfWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/Error.h"
Expand Down Expand Up @@ -455,12 +456,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
Header.MemProfOffset = 0;
Header.BinaryIdOffset = 0;
Header.TemporalProfTracesOffset = 0;
Header.VTableNamesOffset = 0;
int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);

// Only write out all the fields except 'HashOffset', 'MemProfOffset',
// 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
// offset of these fields to allow back patching later.
for (int I = 0; I < N - 4; I++)
// 'BinaryIdOffset', `TemporalProfTracesOffset` and `VTableNamesOffset`. We
// need to remember the offset of these fields to allow back patching later.
for (int I = 0; I < N - 5; I++)
OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);

// Save the location of Header.HashOffset field in \c OS.
Expand All @@ -484,6 +486,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t TemporalProfTracesOffset = OS.tell();
OS.write(0);

uint64_t VTableNamesOffset = OS.tell();
OS.write(0);

// Reserve space to write profile summary data.
uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
Expand Down Expand Up @@ -604,6 +609,43 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
OS.writeByte(0);
}

// if version >= the version with vtable profile metadata.
uint64_t VTableNamesSectionStart = 0;
if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) {
VTableNamesSectionStart = OS.tell();

std::string CompressedVTableNames;

std::vector<std::string> VTableNameStrs;
for (const auto &VTableName : VTableNames.keys()) {
VTableNameStrs.push_back(VTableName.str());
}

if (!VTableNameStrs.empty()) {
if (Error E = collectGlobalObjectNameStrings(
VTableNameStrs, compression::zlib::isAvailable(),
CompressedVTableNames))
return E;
}

uint64_t CompressedStringLen = CompressedVTableNames.length();

// Record the length of compressed string.
OS.write(CompressedStringLen);

// Write the chars in compressed strings.
for (auto &c : CompressedVTableNames)
OS.writeByte(static_cast<uint8_t>(c));

// Pad up to a multiple of 8.
// InstrProfReader could read bytes according to 'CompressedStringLen'.
uint64_t PaddedLength = alignTo(CompressedStringLen, 8);

for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
OS.writeByte(0);
}
}

uint64_t TemporalProfTracesSectionStart = 0;
if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
TemporalProfTracesSectionStart = OS.tell();
Expand Down Expand Up @@ -647,6 +689,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// Patch the Header.TemporalProfTracesOffset (=0 for profiles without
// traces).
{TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
{VTableNamesOffset, &VTableNamesSectionStart, 1},
// Patch the summary data.
{SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
(int)(SummarySize / sizeof(uint64_t))},
Expand Down Expand Up @@ -699,7 +742,8 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
DenseSet<uint64_t> SeenValues;
for (uint32_t I = 0; I < ND; I++)
if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
!SeenValues.insert(VD[I].Value).second)
return make_error<InstrProfError>(instrprof_error::invalid_prof);
}
}
Expand Down Expand Up @@ -747,7 +791,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
OS << ND << "\n";
std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
for (uint32_t I = 0; I < ND; I++) {
if (VK == IPVK_IndirectCallTarget)
if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
<< VD[I].Count << "\n";
else
Expand Down Expand Up @@ -786,6 +830,11 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
}
}

for (const auto &VTableName : VTableNames) {
if (Error E = Symtab.addVTableName(VTableName.getKey()))
return E;
}

if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
writeTextTemporalProfTraceData(OS, Symtab);

Expand Down
173 changes: 173 additions & 0 deletions llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ using namespace llvm;
#define DEBUG_TYPE "instrprof"

namespace llvm {
// Command line option to enable vtable value profiling. Defined in
// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
extern cl::opt<bool> EnableVTableValueProfiling;
// TODO: Remove -debug-info-correlate in next LLVM release, in favor of
// -profile-correlate=debug-info.
cl::opt<bool> DebugInfoCorrelate(
Expand Down Expand Up @@ -196,12 +199,18 @@ class InstrLowerer final {
PerFunctionProfileData() = default;
};
DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
// Key is virtual table variable, value is 'VTableProfData' in the form of
// GlobalVariable.
DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
/// If runtime relocation is enabled, this maps functions to the load
/// instruction that produces the profile relocation bias.
DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
std::vector<GlobalValue *> CompilerUsedVars;
std::vector<GlobalValue *> UsedVars;
std::vector<GlobalVariable *> ReferencedNames;
// The list of virtual table variables of which the VTableProfData is
// collected.
std::vector<GlobalVariable *> ReferencedVTables;
GlobalVariable *NamesVar = nullptr;
size_t NamesSize = 0;

Expand Down Expand Up @@ -294,9 +303,15 @@ class InstrLowerer final {
/// Create INSTR_PROF_DATA variable for counters and bitmaps.
void createDataVariable(InstrProfCntrInstBase *Inc);

/// Get the counters for virtual table values, creating them if necessary.
void getOrCreateVTableProfData(GlobalVariable *GV);

/// Emit the section with compressed function names.
void emitNameData();

/// Emit the section with compressed vtable names.
void emitVTableNames();

/// Emit value nodes section for value profiling.
void emitVNodes();

Expand Down Expand Up @@ -740,6 +755,15 @@ bool InstrLowerer::lower() {
}
}

if (EnableVTableValueProfiling) {
for (GlobalVariable &GV : M.globals()) {
// Global variables with type metadata are virtual table variables.
if (GV.hasMetadata(LLVMContext::MD_type)) {
getOrCreateVTableProfData(&GV);
}
}
}

for (Function &F : M)
MadeChange |= lowerIntrinsics(&F);

Expand All @@ -753,6 +777,7 @@ bool InstrLowerer::lower() {

emitVNodes();
emitNameData();
emitVTableNames();

// Emit runtime hook for the cases where the target does not unconditionally
// require pulling in profile runtime, and coverage is enabled on code that is
Expand Down Expand Up @@ -1220,6 +1245,129 @@ void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn,
GV->setLinkage(GlobalValue::InternalLinkage);
}

static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
if (!profDataReferencedByCode(*GV->getParent()))
return false;

if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
!GV->hasAvailableExternallyLinkage())
return true;

// This avoids the profile data from referencing internal symbols in
// COMDAT.
if (GV->hasLocalLinkage() && GV->hasComdat())
return false;

return true;
}

// FIXME: Does symbolic relocation from 'getFuncAddrForProfData' matter here?
static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
auto *Int8PtrTy = PointerType::getUnqual(GV->getContext());

// Store a nullptr in __profvt_ if a real address shouldn't be used.
if (!shouldRecordVTableAddr(GV))
return ConstantPointerNull::get(Int8PtrTy);

return ConstantExpr::getBitCast(GV, Int8PtrTy);
}

void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
assert(!DebugInfoCorrelate &&
"Value profiling is not supported with lightweight instrumentation");
if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
return;

if (GV->getName().starts_with("llvm.") ||
GV->getName().starts_with("__llvm") ||
GV->getName().starts_with("__prof"))
return;

// VTableProfData already created
auto It = VTableDataMap.find(GV);
if (It != VTableDataMap.end() && It->second)
return;

GlobalValue::LinkageTypes Linkage = GV->getLinkage();
GlobalValue::VisibilityTypes Visibility = GV->getVisibility();

// This is to keep consistent with per-function profile data
// for correctness.
if (TT.isOSBinFormatXCOFF()) {
Linkage = GlobalValue::InternalLinkage;
Visibility = GlobalValue::DefaultVisibility;
}

LLVMContext &Ctx = M.getContext();
Type *DataTypes[] = {
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
#include "llvm/ProfileData/InstrProfData.inc"
};

auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));

// Used by INSTR_PROF_VTABLE_DATA MACRO
Constant *VTableAddr = getVTableAddrForProfData(GV);
const std::string PGOVTableName = getPGOName(*GV);
// Record the length of the vtable. This is needed since vtable pointers
// loaded from C++ objects might be from the middle of a vtable definition.
uint32_t VTableSizeVal =
M.getDataLayout().getTypeAllocSize(GV->getValueType());

Constant *DataVals[] = {
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
#include "llvm/ProfileData/InstrProfData.inc"
};

std::string VarName = getInstrProfVTableVarPrefix().str() + PGOVTableName;
auto *Data =
new GlobalVariable(M, DataTy, false /* constant */, Linkage,
ConstantStruct::get(DataTy, DataVals), VarName);

Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
Data->setAlignment(Align(8));

const bool NeedComdat = needsComdatForCounter(*GV, M);

// GV is the data structure to record vtable information.
// Place the global variable for per-vtable profile data in a comdat group
// if the associated vtable definition is a COMDAT. This makes sure only one
// copy of the variable for the vtable will be emitted after linking.
auto MaybeSetComdat = [&](GlobalVariable *GV, StringRef GroupName) {
bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
if (UseComdat) {
// Create a new comdat group using the name of the global variable as
// opposed to using the comdat group of the vtable.
Comdat *C = M.getOrInsertComdat(GroupName);
// For ELF, when not using COMDAT, put the vtable profile data into a
// nodeduplicate COMDAT which is lowered to a zero-flag zero group.
// This allows -z -start-stop-gc to discard the entire group when the
// vtable def is discarded.
if (!NeedComdat)
C->setSelectionKind(Comdat::NoDeduplicate);
GV->setComdat(C);
// COFF doesn't allow the comdat group leader to have private linkage, so
// upgrade private linkage to internal linkage to produce a symbol table
// entry.
if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) {
GV->setLinkage(GlobalValue::InternalLinkage);
}
return;
}
};

MaybeSetComdat(Data, Data->getName());

VTableDataMap[GV] = Data;

ReferencedVTables.push_back(GV);

// VTable <Hash, Addr> is used by runtime but not referenced by other
// sections. Conservatively mark it linker retained.
UsedVars.push_back(Data);
}

GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
InstrProfSectKind IPSK) {
GlobalVariable *NamePtr = Inc->getName();
Expand Down Expand Up @@ -1633,6 +1781,31 @@ void InstrLowerer::emitNameData() {
NamePtr->eraseFromParent();
}

void InstrLowerer::emitVTableNames() {
if (!EnableVTableValueProfiling || ReferencedVTables.empty())
return;

// Collect the PGO names of referenced vtables and compress them.
std::string CompressedVTableNames;
if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,
DoInstrProfNameCompression)) {
report_fatal_error(Twine(toString(std::move(E))), false);
}

auto &Ctx = M.getContext();
auto *VTableNamesVal = ConstantDataArray::getString(
Ctx, StringRef(CompressedVTableNames), false /* AddNull */);
GlobalVariable *VTableNamesVar =
new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
GlobalValue::PrivateLinkage, VTableNamesVal,
getInstrProfVTableNamesVarName());
VTableNamesVar->setSection(
getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));
VTableNamesVar->setAlignment(Align(1));
// Make VTableNames linker retained.
UsedVars.push_back(VTableNamesVar);
}

void InstrLowerer::emitRegistration() {
if (!needsRuntimeRegistrationOfSectionRange(TT))
return;
Expand Down
7 changes: 7 additions & 0 deletions llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,11 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;

extern cl::opt<bool> DebugInfoCorrelate;

// Command line option to enable vtable value profiling. Defined in
// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
extern cl::opt<bool> EnableVTableValueProfiling;
extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
} // namespace llvm

Expand Down Expand Up @@ -581,6 +586,8 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
NumOfPGOBB += MST.bbInfoSize();
ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);
if (EnableVTableValueProfiling)
ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
Expand Down
36 changes: 33 additions & 3 deletions llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,39 @@ public:
}
};

///------------------------ VirtualTableValueProfilingPlugin
///------------------------
class VTableProfilingPlugin {
Function &F;

public:
static constexpr InstrProfValueKind Kind = IPVK_VTableTarget;

VTableProfilingPlugin(Function &Fn, TargetLibraryInfo &TLI) : F(Fn) {}

void run(std::vector<CandidateInfo> &Candidates) {
std::vector<Instruction *> Result = findVTableAddrs(F);
for (Instruction *I : Result) {
Instruction *InsertPt = I->getNextNonDebugInstruction();
// When finding an insertion point, keep PHI and EH pad instructions
// before vp intrinsics. This is similar to
// `BasicBlock::getFirstInsertionPt`.
while (InsertPt && (dyn_cast<PHINode>(InsertPt) || InsertPt->isEHPad()))
InsertPt = InsertPt->getNextNonDebugInstruction();
// Skip instrumentating the value if InsertPt is the last instruction.
// FIXME: Set InsertPt to the end of basic block to instrument the value
// if InsertPt is the last instruction.
if (InsertPt == nullptr)
continue;

Instruction *AnnotatedInst = I;
Candidates.emplace_back(CandidateInfo{I, InsertPt, AnnotatedInst});
}
}
};

///----------------------- Registration of the plugins -------------------------
/// For now, registering a plugin with the ValueProfileCollector is done by
/// adding the plugin type to the VP_PLUGIN_LIST macro.
#define VP_PLUGIN_LIST \
MemIntrinsicPlugin, \
IndirectCallPromotionPlugin
#define VP_PLUGIN_LIST \
MemIntrinsicPlugin, IndirectCallPromotionPlugin, VTableProfilingPlugin
8 changes: 4 additions & 4 deletions llvm/test/Instrumentation/InstrProfiling/coverage.ll
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ target triple = "aarch64-unknown-linux-gnu"

@__profn_foo = private constant [3 x i8] c"foo"
; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64),
; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64),
@__profn_bar = private constant [3 x i8] c"bar"
; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64),
; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64),

; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names"
; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames"
Expand Down
Binary file not shown.
Binary file not shown.
4 changes: 2 additions & 2 deletions llvm/test/Transforms/PGOProfile/comdat_internal.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ $foo = comdat any
; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat
; CHECK-NOT: __profn__stdin__foo
; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null
; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null
; CHECK-NOT: @foo
; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8
; CHECK: @__llvm_prf_nm
; CHECK: @llvm.compiler.used

Expand Down
98 changes: 98 additions & 0 deletions llvm/test/Transforms/PGOProfile/vtable_profile.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
; RUN: opt < %s -passes=pgo-instr-gen -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -enable-vtable-value-profiling -S | FileCheck %s --check-prefix=LOWER

; __llvm_prf_vnm stores zlib-compressed vtable names.
; REQUIRES: zlib

source_filename = "vtable_local.ll"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

; The test IR is generated based on the following C++ program.
; Base1 has external linkage and Base2 has local linkage.
; class Derived uses multiple inheritance so its virtual table
; global variable contains two vtables. func1 is loaded from
; the vtable compatible with class Base1, and func2 is loaded
; from the vtable compatible with class Base2.

; class Base1 {
; public:
; virtual int func1(int a) ;
; };
;
; namespace {
; class Base2 {
; public:
; __attribute__((noinline)) virtual int func2(int a) {
; return a;
; }
; };
; }

; class Derived : public Base1, public Base2 {
; public:
; Derived(int c) : v(c) {}
; private:
; int v;
; };
;
; Derived* createType();

; int func(int a) {
; Derived* d = createType();
; return d->func2(a) + d->func1(a);
; }

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

@_ZTV7Derived = constant { [3 x ptr], [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei], [3 x ptr] [ptr inttoptr (i64 -8 to ptr), ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !0, !type !3, !type !6, !type !8, !type !10
@_ZTV5Base1 = available_externally constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN5Base15func1Ei] }, !type !0
@_ZTVN12_GLOBAL__N_15Base2E = internal constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN12_GLOBAL__N_15Base25func2Ei] }, !type !11, !type !8; !vcall_visibility !12
@llvm.compiler.used = appending global [1 x ptr] [ptr @_ZTV5Base1], section "llvm.metadata"

; GEN: __llvm_profile_raw_version = comdat any
; GEN: __llvm_profile_raw_version = hidden constant i64 72057594037927946, comdat
; GEN: __profn__Z4funci = private constant [8 x i8] c"_Z4funci"

; LOWER: $__profvt__ZTV7Derived = comdat nodeduplicate
; LOWER: $"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = comdat nodeduplicate
; LOWER: @__profvt__ZTV7Derived = global { i64, ptr, i32 } { i64 -4576307468236080025, ptr @_ZTV7Derived, i32 48 }, section "__llvm_prf_vtab", comdat, align 8
; LOWER: @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E" = internal global { i64, ptr, i32 } { i64 1419990121885302679, ptr @_ZTVN12_GLOBAL__N_15Base2E, i32 24 }, section "__llvm_prf_vtab", comdat, align 8
; LOWER: @__llvm_prf_vnm = private constant [64 x i8] c"7>x\DA\8B\8F\0A\093wI-\CA,KMa,+IL\CAI\8D\CF\C9ON\CC\D1\CB\C9\B1\8E\07J\FA\19\1A\C5\BB\FB\F8;9\FA\C4\C7\FB\C5\1B\9A:%\16\A7\1A\B9\02\00\19:\12o", section "__llvm_prf_vtabnames", align 1
; LOWER: @llvm.used = appending global [5 x ptr] [ptr @__profvt__ZTV7Derived, ptr @"__profvt_vtable_local.ll;_ZTVN12_GLOBAL__N_15Base2E", ptr @__llvm_prf_vnodes, ptr @__llvm_prf_nm, ptr @__llvm_prf_vnm], section "llvm.metadata"

define i32 @_Z4funci(i32 %a) {
entry:
%call = call ptr @_Z10createTypev()
%add.ptr = getelementptr inbounds i8, ptr %call, i64 8
%vtable = load ptr, ptr %add.ptr
; GEN: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64
; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash:[0-9]+]], i64 [[P1]], i32 2, i32 0)
; LOWER: [[P1:%[0-9]+]] = ptrtoint ptr %vtable to i64
; LOWER: call void @__llvm_profile_instrument_target(i64 [[P1]], ptr @__profd__Z4funci, i32 2)
%vfunc1 = load ptr, ptr %vtable
%call1 = call i32 %vfunc1(ptr %add.ptr, i32 %a)
%vtable2 = load ptr, ptr %call
; GEN: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64
; GEN: call void @llvm.instrprof.value.profile(ptr @__profn__Z4funci, i64 [[CFGHash]], i64 [[P2]], i32 2, i32 1)
; LOWER: [[P2:%[0-9]+]] = ptrtoint ptr %vtable2 to i64
; LOWER: call void @__llvm_profile_instrument_target(i64 [[P2]], ptr @__profd__Z4funci, i32 3)
%vfunc2 = load ptr, ptr %vtable2
%call4 = call i32 %vfunc2(ptr %call, i32 %a)
%add = add nsw i32 %call1, %call4
ret i32 %add
}

declare ptr @_Z10createTypev()
declare i32 @_ZN12_GLOBAL__N_15Base25func2Ei(ptr %this, i32 %a)
declare i32 @_ZN5Base15func1Ei(ptr, i32)

!0 = !{i64 16, !"_ZTS5Base1"}
!3 = !{i64 16, !"_ZTS7Derived"}
!6 = !{i64 40, !7}
!7 = distinct !{}
!8 = !{i64 16, !9}
!9 = distinct !{}
!10 = !{i64 40, !9}
!11 = !{i64 16, !7}
Binary file modified llvm/test/tools/llvm-profdata/Inputs/c-general.profraw
Binary file not shown.
Binary file modified llvm/test/tools/llvm-profdata/Inputs/compressed.profraw
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/bash

if [ $# -lt 1 ]; then
echo "Path to clang++ required!"
echo "Usage: update_vtable_value_prof_inputs.sh /path/to/updated/clang++"
exit 1
else
CLANG=$1
fi


# Remember current directory.
CURDIR=$PWD

# Allows the script to be invoked from other directories.
OUTDIR=$(dirname $(realpath -s $0))
echo $OUTDIR

cd $OUTDIR

# vtable_prof.cc has the following class hierarchy:
# class Base
# ├── class Derived1
# └── class Derived2
# Derived1 is a class in the global namespace and Derived2 is in anonymous
# namespace for test coverage. Overridden virtual methods are annotated as
# `noinline` so the callsite remains indirect calls for testing purposes.
cat > vtable_prof.cc << EOF
#include <cstdlib>
#include <cstdio>
class Base {
public:
virtual int func1(int a, int b) = 0;
virtual int func2(int a, int b) = 0;
};
class Derived1 : public Base {
public:
__attribute__((noinline))
int func1(int a, int b) override
{
return a + b;
}
__attribute__((noinline))
int func2(int a, int b) override {
return a * b;
}
};
namespace {
class Derived2 : public Base {
public:
__attribute__((noinline))
int func1(int a, int b) override {
return a - b;
}
__attribute__((noinline))
int func2(int a, int b) override {
return a * (a - b);
}
};
} // namespace
__attribute__((noinline)) Base* createType(int a) {
Base* base = nullptr;
if (a % 4 == 0)
base = new Derived1();
else
base = new Derived2();
return base;
}
int main(int argc, char** argv) {
int sum = 0;
for (int i = 0; i < 1000; i++) {
int a = rand();
int b = rand();
Base* ptr = createType(i);
sum += ptr->func1(a, b) + ptr->func2(b, a);
}
printf("sum is %d\n", sum);
return 0;
}
EOF


# Clean up temporary files on exit and return to original directory.
cleanup() {
rm -f vtable_prof
rm -f vtable_prof.cc
cd $CURDIR
}
trap cleanup EXIT

FLAGS="-fuse-ld=lld -O2 -g -fprofile-generate=. -mllvm -enable-vtable-value-profiling"

${CLANG} ${FLAGS} vtable_prof.cc -o vtable_prof
env LLVM_PROFILE_FILE=vtable-value-prof-basic.profraw ./vtable_prof
Binary file not shown.
73 changes: 73 additions & 0 deletions llvm/test/tools/llvm-profdata/Inputs/vtable-value-prof.proftext
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# IR level Instrumentation Flag
:ir
/path/to/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii
# Func Hash:
742261418966908927
# Num Counters:
1
# Counter Values:
750

/path/to/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii
# Func Hash:
742261418966908927
# Num Counters:
1
# Counter Values:
750

_Z10createTypei
# Func Hash:
146835647075900052
# Num Counters:
2
# Counter Values:
750
250

_ZN8Derived15func1Eii
# Func Hash:
742261418966908927
# Num Counters:
1
# Counter Values:
250

_ZN8Derived15func2Eii
# Func Hash:
742261418966908927
# Num Counters:
1
# Counter Values:
250

main
# Func Hash:
1124236338992350536
# Num Counters:
2
# Counter Values:
1000
1
# Num Value Kinds:
2
# ValueKind = IPVK_IndirectCallTarget:
0
# NumValueSites:
2
2
/path/to/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750
_ZN8Derived15func1Eii:250
2
/path/to/vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750
_ZN8Derived15func2Eii:250
# ValueKind = IPVK_VTableTarget:
2
# NumValueSites:
2
2
/path/to/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
_ZTV8Derived1:250
2
/path/to/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
_ZTV8Derived1:250
6 changes: 5 additions & 1 deletion llvm/test/tools/llvm-profdata/binary-ids-padding.test
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)

RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
// There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes.
// 2 * 8 binary ID sizes
// + 2 * 20 binary IDs (of size 20)
Expand All @@ -32,6 +34,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
Expand Down
4 changes: 3 additions & 1 deletion llvm/test/tools/llvm-profdata/large-binary-id-size.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
Expand All @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Check for a corrupted size being too large past the end of the file.
RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)

RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
Expand All @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Data Section
//
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)

RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
Expand All @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Data Section
//
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
// INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
// INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)

RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
Expand All @@ -26,6 +28,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Data Section
//
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw
// We should fail on this because the binary IDs is not a multiple of 8 bytes.
RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
Expand All @@ -10,6 +10,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw

// Binary IDs - There are only two in this case that are 20 bytes.
RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t

RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s

Expand Down
11 changes: 6 additions & 5 deletions llvm/test/tools/llvm-profdata/raw-32-bits-be.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Header
RUN: printf '\377lprofR\201' > %t
RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\12' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
Expand All @@ -12,6 +13,8 @@ RUN: printf '\0\0\0\0\1\0\0\0' >> %t
RUN: printf '\0\0\0\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t

RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
Expand All @@ -20,9 +23,8 @@ RUN: printf '\3\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\1' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\3' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\344\023\165\112\031\035\265\067' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
Expand All @@ -31,9 +33,8 @@ RUN: printf '\2\xff\xff\xd3' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\2' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\0\0\0\0\0\0\0\023' >> %t
RUN: printf '\0\0\0\0\0\0\0\067' >> %t
Expand Down
10 changes: 5 additions & 5 deletions llvm/test/tools/llvm-profdata/raw-32-bits-le.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\201Rforpl\377' > %t
RUN: printf '\11\0\0\0\0\0\0\0' >> %t
RUN: printf '\12\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
Expand All @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\0\0\0' >> %t
RUN: printf '\0\0\0\3\0\0\0\0' >> %t
RUN: printf '\0\0\0\2\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t

RUN: printf '\254\275\030\333\114\302\370\134' >> %t
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
Expand All @@ -20,9 +22,8 @@ RUN: printf '\0\0\0\3' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\1\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\3\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
Expand All @@ -31,9 +32,8 @@ RUN: printf '\xd3\xff\xff\2' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\2\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\1\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\023\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\0\0\0\0\0\0\0' >> %t
Expand Down
10 changes: 5 additions & 5 deletions llvm/test/tools/llvm-profdata/raw-64-bits-be.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\377lprofr\201' > %t
RUN: printf '\0\0\0\0\0\0\0\11' >> %t
RUN: printf '\0\0\0\0\0\0\0\12' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\2' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
Expand All @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\4\0\0' >> %t
RUN: printf '\0\0\0\3\0\4\0\0' >> %t
RUN: printf '\0\0\0\2\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t

RUN: printf '\134\370\302\114\333\030\275\254' >> %t
RUN: printf '\0\0\0\0\0\0\0\1' >> %t
Expand All @@ -20,9 +22,8 @@ RUN: printf '\0\0\0\3\0\4\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\3' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\344\023\165\112\031\035\265\067' >> %t
RUN: printf '\0\0\0\0\0\0\0\02' >> %t
Expand All @@ -31,9 +32,8 @@ RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\02' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\1' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\0\0\0\0\0\0\0\023' >> %t
RUN: printf '\0\0\0\0\0\0\0\067' >> %t
Expand Down
10 changes: 5 additions & 5 deletions llvm/test/tools/llvm-profdata/raw-64-bits-le.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t
RUN: printf '\11\0\0\0\0\0\0\0' >> %t
RUN: printf '\12\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\2\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
Expand All @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t
RUN: printf '\0\0\4\0\3\0\0\0' >> %t
RUN: printf '\0\0\4\0\2\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t

RUN: printf '\254\275\030\333\114\302\370\134' >> %t
RUN: printf '\1\0\0\0\0\0\0\0' >> %t
Expand All @@ -20,9 +22,8 @@ RUN: printf '\0\0\4\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\1\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\3\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\067\265\035\031\112\165\023\344' >> %t
RUN: printf '\02\0\0\0\0\0\0\0' >> %t
Expand All @@ -31,9 +32,8 @@ RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\02\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t
RUN: printf '\0\0\0\0\0\0\0\0' >> %t
RUN: printf '\1\0\0\0' >> %t
RUN: printf '\0\0\0\0' >> %t

RUN: printf '\023\0\0\0\0\0\0\0' >> %t
RUN: printf '\067\0\0\0\0\0\0\0' >> %t
Expand Down
8 changes: 6 additions & 2 deletions llvm/test/tools/llvm-profdata/raw-two-profiles.test
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
RUN: printf '\201rforpl\377' > %t-foo.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
Expand All @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw

RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
Expand All @@ -26,7 +28,7 @@ RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw
RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw

RUN: printf '\201rforpl\377' > %t-bar.profraw
RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\12\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
Expand All @@ -39,6 +41,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw

RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw
RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw
Expand Down
124 changes: 124 additions & 0 deletions llvm/test/tools/llvm-profdata/vtable-value-prof-basic.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
To update the inputs used below, run
Inputs/update_vtable_value_prof_inputs.sh /path/to/updated/clang++

; Raw profiles stores zlib-compressed vtable names. Raw profile reader needs
; to decompress them.
; REQUIRES: zlib

; RUN: rm -rf %t && mkdir %t && cd %t

Show profile data from raw profiles.
RUN: llvm-profdata show --function=main --ic-targets --show-vtables %p/Inputs/vtable-value-prof-basic.profraw | FileCheck %s --check-prefix=RAW

Generate indexed profile from raw profile and show the data.
RUN: llvm-profdata merge %p/Inputs/vtable-value-prof-basic.profraw -o indexed.profdata
RUN: llvm-profdata show --function=main --ic-targets --show-vtables indexed.profdata | FileCheck %s --check-prefix=INDEXED

Generate text profile from raw profile and show the data.
RUN: llvm-profdata merge --text %p/Inputs/vtable-value-prof-basic.profraw -o vtable-value-prof-basic.proftext
RUN: llvm-profdata show --function=main --ic-targets --show-vtables --text vtable-value-prof-basic.proftext | FileCheck %s --check-prefix=ICTEXT

RAW: Counters:
RAW-NEXT: main:
RAW-NEXT: Hash: 0x0f9a16fe6d398548
RAW-NEXT: Counters: 2
RAW-NEXT: Indirect Call Site Count: 2
RAW-NEXT: Number of instrumented vtables: 2
RAW-NEXT: Indirect Target Results:
RAW-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
RAW-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
RAW-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%)
RAW-NEXT: VTable Results:
RAW-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
RAW-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
RAW-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
RAW-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
RAW-NEXT: Instrumentation level: IR entry_first = 0
RAW-NEXT: Functions shown: 1
RAW-NEXT: Total functions: 6
RAW-NEXT: Maximum function count: 1000
RAW-NEXT: Maximum internal block count: 250
RAW-NEXT: Statistics for indirect call sites profile:
RAW-NEXT: Total number of sites: 2
RAW-NEXT: Total number of sites with values: 2
RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2
RAW-NEXT: Statistics for vtable profile:
RAW-NEXT: Total number of sites: 2
RAW-NEXT: Total number of sites with values: 2
RAW-NEXT: Total number of profiled values: 4
RAW-NEXT: Value sites histogram:
RAW-NEXT: NumTargets, SiteCount
RAW-NEXT: 2, 2


INDEXED: Counters:
INDEXED-NEXT: main:
INDEXED-NEXT: Hash: 0x0f9a16fe6d398548
INDEXED-NEXT: Counters: 2
INDEXED-NEXT: Indirect Call Site Count: 2
INDEXED-NEXT: Number of instrumented vtables: 2
INDEXED-NEXT: Indirect Target Results:
INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZN8Derived15func1Eii, 250 ] (25.00%)
INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZN8Derived15func2Eii, 250 ] (25.00%)
INDEXED-NEXT: VTable Results:
INDEXED-NEXT: [ 0, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
INDEXED-NEXT: [ 0, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: [ 1, {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
INDEXED-NEXT: [ 1, _ZTV8Derived1, 250 ] (25.00%)
INDEXED-NEXT: Instrumentation level: IR entry_first = 0
INDEXED-NEXT: Functions shown: 1
INDEXED-NEXT: Total functions: 6
INDEXED-NEXT: Maximum function count: 1000
INDEXED-NEXT: Maximum internal block count: 250
INDEXED-NEXT: Statistics for indirect call sites profile:
INDEXED-NEXT: Total number of sites: 2
INDEXED-NEXT: Total number of sites with values: 2
INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
INDEXED-NEXT: NumTargets, SiteCount
INDEXED-NEXT: 2, 2
INDEXED-NEXT: Statistics for vtable profile:
INDEXED-NEXT: Total number of sites: 2
INDEXED-NEXT: Total number of sites with values: 2
INDEXED-NEXT: Total number of profiled values: 4
INDEXED-NEXT: Value sites histogram:
INDEXED-NEXT: NumTargets, SiteCount
INDEXED-NEXT: 2, 2

ICTEXT: :ir
ICTEXT: main
ICTEXT: # Func Hash:
ICTEXT: 1124236338992350536
ICTEXT: # Num Counters:
ICTEXT: 2
ICTEXT: # Counter Values:
ICTEXT: 1000
ICTEXT: 1
ICTEXT: # Num Value Kinds:
ICTEXT: 2
ICTEXT: # ValueKind = IPVK_IndirectCallTarget:
ICTEXT: 0
ICTEXT: # NumValueSites:
ICTEXT: 2
ICTEXT: 2
ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func1Eii:750
ICTEXT: _ZN8Derived15func1Eii:250
ICTEXT: 2
ICTEXT: {{.*}}vtable_prof.cc;_ZN12_GLOBAL__N_18Derived25func2Eii:750
ICTEXT: _ZN8Derived15func2Eii:250
ICTEXT: # ValueKind = IPVK_VTableTarget:
ICTEXT: 2
ICTEXT: # NumValueSites:
ICTEXT: 2
ICTEXT: 2
ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
ICTEXT: _ZTV8Derived1:250
ICTEXT: 2
ICTEXT: {{.*}}vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E:750
ICTEXT: _ZTV8Derived1:250
16 changes: 16 additions & 0 deletions llvm/test/tools/llvm-profdata/vtable-value-prof.proftext
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# RUN: llvm-profdata show --function=main --show-vtables %p/Inputs/vtable-value-prof.proftext | FileCheck %s

# CHECK: Counters:
# CHECK: main:
# CHECK: Hash: 0x0f9a16fe6d398548
# CHECK: Counters: 2
# CHECK: VTable Results:
# CHECK: [ 0, /path/to/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
# CHECK: [ 0, _ZTV8Derived1, 250 ] (25.00%)
# CHECK: [ 1, /path/to/vtable_prof.cc;_ZTVN12_GLOBAL__N_18Derived2E, 750 ] (75.00%)
# CHECK: [ 1, _ZTV8Derived1, 250 ] (25.00%)
# CHECK: Instrumentation level: IR entry_first = 0
# CHECK: Functions shown: 1
# CHECK: Total functions: 6
# CHECK: Maximum function count: 1000
# CHECK: Maximum internal block count: 250
30 changes: 28 additions & 2 deletions llvm/tools/llvm-profdata/llvm-profdata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,9 @@ cl::opt<bool> ShowIndirectCallTargets(
"ic-targets", cl::init(false),
cl::desc("Show indirect call site target values for shown functions"),
cl::sub(ShowSubcommand));
cl::opt<bool> ShowVTables("show-vtables", cl::init(false),
cl::desc("Show vtable names for shown functions"),
cl::sub(ShowSubcommand));
cl::opt<bool> ShowMemOPSizes(
"memop-sizes", cl::init(false),
cl::desc("Show the profiled sizes of the memory intrinsic calls "
Expand Down Expand Up @@ -715,6 +718,13 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
});
}

const InstrProfSymtab &symtab = Reader->getSymtab();
const auto &VTableNames = symtab.getVTableNames();

for (const auto &kv : VTableNames) {
WC->Writer.addVTableName(kv.getKey());
}

if (Reader->hasTemporalProfile()) {
auto &Traces = Reader->getTemporalProfTraces(Input.Weight);
if (!Traces.empty())
Expand Down Expand Up @@ -1288,8 +1298,8 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
BodySample.second.getSamples());
for (const auto &Target : BodySample.second.getCallTargets()) {
Result.addCalledTargetSamples(BodySample.first.LineOffset,
MaskedDiscriminator,
Remapper(Target.first), Target.second);
MaskedDiscriminator, Remapper(Target.first),
Target.second);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
Expand Down Expand Up @@ -2750,6 +2760,10 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
OS << " Indirect Call Site Count: "
<< Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n";

if (ShowVTables)
OS << " Number of instrumented vtables: "
<< Func.getNumValueSites(IPVK_VTableTarget) << "\n";

uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize);
if (ShowMemOPSizes && NumMemOPCalls > 0)
OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
Expand All @@ -2771,6 +2785,13 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
&(Reader->getSymtab()));
}

if (ShowVTables) {
OS << " VTable Results:\n";
traverseAllValueSites(Func, IPVK_VTableTarget,
VPStats[IPVK_VTableTarget], OS,
&(Reader->getSymtab()));
}

if (ShowMemOPSizes && NumMemOPCalls > 0) {
OS << " Memory Intrinsic Size Results:\n";
traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS,
Expand Down Expand Up @@ -2819,6 +2840,11 @@ static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
VPStats[IPVK_IndirectCallTarget]);
}

if (ShownFunctions && ShowVTables) {
OS << "Statistics for vtable profile:\n";
showValueSitesStats(OS, IPVK_VTableTarget, VPStats[IPVK_VTableTarget]);
}

if (ShownFunctions && ShowMemOPSizes) {
OS << "Statistics for memory intrinsic calls sizes profile:\n";
showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]);
Expand Down
Loading