46 changes: 46 additions & 0 deletions compiler-rt/test/profile/instrprof-binary-correlate.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// REQUIRES: linux || windows
// Default
// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal
// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw
// RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report
// RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show

// With -profile-correlate=binary flag
// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe
// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw
// RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report
// RUN: llvm-cov show --instr-profile=%t-1.profdata %t-1.exe > %t-1.show
// RUN: diff %t.normal.profdata %t-1.profdata
// RUN: diff %t.normal.report %t-1.report
// RUN: diff %t.normal.show %t-1.show

// Strip above binary and run
// RUN: llvm-strip %t-1.exe -o %t-2.exe
// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe
// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw
// RUN: llvm-cov report --instr-profile=%t-2.profdata %t-1.exe > %t-2.report
// RUN: llvm-cov show --instr-profile=%t-2.profdata %t-1.exe > %t-2.show
// RUN: diff %t.normal.profdata %t-2.profdata
// RUN: diff %t.normal.report %t-2.report
// RUN: diff %t.normal.show %t-2.show

// Online merging.
// RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal
// RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal
// RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw
// RUN: llvm-cov report --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.report
// RUN: llvm-cov show --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.show

// RUN: rm -rf %t.profdir && mkdir %t.profdir
// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe
// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe
// RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir
// RUN: llvm-cov report --instr-profile=%t-4.profdata %t-1.exe > %t-4.report
// RUN: llvm-cov show --instr-profile=%t-4.profdata %t-1.exe > %t-4.show
// RUN: diff %t.normal.merged.profdata %t-4.profdata
// RUN: diff %t.normal.merged.report %t-4.report
// RUN: diff %t.normal.merged.show %t-4.show

// TODO: After adding support for binary ID, test binaries with different binary IDs.
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
// The test uses pthread barriers which are not available on Darwin.
// UNSUPPORTED: darwin

// FIXME: It probably hangs on this platform.
// UNSUPPORTED: ppc

// Forking in multithread environment is unsupported. However we already have
// some workarounds, and will add more, so this is the test.
// The test try to check two things:
Expand Down
2 changes: 2 additions & 0 deletions lldb/include/lldb/Host/Editline.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ using EditLineCharType = char;
// to wchar_t. It is not possible to detect differentiate between the two
// versions exactly, but this is a pretty good approximation and allows us to
// build against almost any editline version out there.
// It does, however, require extra care when invoking el_getc, as the type
// of the input is a single char buffer, but the callback will write a wchar_t.
#if LLDB_EDITLINE_USE_WCHAR || defined(EL_CLIENTDATA) || LLDB_HAVE_EL_RFUNC_T
using EditLineGetCharType = wchar_t;
#else
Expand Down
10 changes: 8 additions & 2 deletions lldb/source/Host/common/Editline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -978,8 +978,14 @@ void Editline::DisplayCompletions(
break;

fprintf(editline.m_output_file, "More (Y/n/a): ");
char reply = 'n';
int got_char = el_getc(editline.m_editline, &reply);
// The type for the output and the type for the parameter are different,
// to allow interoperability with older versions of libedit. The container
// for the reply must be as wide as what our implementation is using,
// but libedit may use a narrower type depending on the build
// configuration.
EditLineGetCharType reply = L'n';
int got_char = el_wgetc(editline.m_editline,
reinterpret_cast<EditLineCharType *>(&reply));
// Check for a ^C or other interruption.
if (editline.m_editor_status == EditorStatus::Interrupted) {
editline.m_editor_status = EditorStatus::Editing;
Expand Down
14 changes: 0 additions & 14 deletions lldb/source/Target/Thread.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -717,17 +717,6 @@ void Thread::DidResume() {

void Thread::DidStop() { SetState(eStateStopped); }

#define CHECK_BEFORE_POP_PLAN \
{ \
uint32_t i = 0; \
ThreadPlanSP p; \
while ((p = GetPlans().GetPlanByIndex(i, false))) \
i++; \
(void)i; \
assert(i != 1 && \
"Cannot pop plan when there is only one plan (the base plan)"); \
}

bool Thread::ShouldStop(Event *event_ptr) {
ThreadPlan *current_plan = GetCurrentPlan();

Expand Down Expand Up @@ -842,7 +831,6 @@ bool Thread::ShouldStop(Event *event_ptr) {
do {
if (should_stop)
current_plan->WillStop();
CHECK_BEFORE_POP_PLAN;
PopPlan();
} while ((current_plan = GetCurrentPlan()) != prev_plan_ptr);
// Now, if the responsible plan was not "Okay to discard" then
Expand Down Expand Up @@ -895,7 +883,6 @@ bool Thread::ShouldStop(Event *event_ptr) {
// If a Controlling Plan wants to stop, we let it. Otherwise, see if
// the plan's parent wants to stop.

CHECK_BEFORE_POP_PLAN;
PopPlan();
if (should_stop && current_plan->IsControllingPlan() &&
!current_plan->OkayToDiscard()) {
Expand Down Expand Up @@ -944,7 +931,6 @@ bool Thread::ShouldStop(Event *event_ptr) {
// plan is complete but does not explain the stop (example: step to a
// line with breakpoint), let us move the plan to
// completed_plan_stack anyway
CHECK_BEFORE_POP_PLAN;
PopPlan();
} else
DiscardPlan();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ def copy_to_main(self, src, dst):
# The rerun tests indicate rerunning on Windows doesn't really work, so
# this one won't either.
@skipIfWindows
# On Arm and AArch64 Linux, this test attempts to pop a thread plan when
# we only have the base plan remaining. Skip it until we can figure out
# the bug this is exposing.
@skipIf(oslist=["linux"], archs=["arm", "aarch64"])
def test_OneTargetOneDebugger(self):
self.do_test(True, True)

Expand All @@ -50,11 +54,6 @@ def test_OneTargetTwoDebuggers(self):
self.do_test(True, False)

def do_test(self, one_target, one_debugger):
# Here to debug flakiness on Arm, remove later!
log_cmd_result = lldb.SBCommandReturnObject()
interp = self.dbg.GetCommandInterpreter()
interp.HandleCommand("log enable lldb step", log_cmd_result)

# Make sure that if we have one target, and we run, then
# change the binary and rerun, the binary (and any .o files
# if using dwarf in .o file debugging) get removed from the
Expand Down
15 changes: 11 additions & 4 deletions llvm/docs/CommandGuide/llvm-profdata.rst
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,14 @@ OPTIONS
.. option:: --debug-info=<path>

Specify the executable or ``.dSYM`` that contains debug info for the raw profile.
When ``-debug-info-correlate`` was used for instrumentation, use this option
to correlate the raw profile.
When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used
for instrumentation, use this option to correlate the raw profile.

.. option:: --binary-file=<path>

Specify the executable that contains profile data and profile name sections for
the raw profile. When ``-profile-correlate=binary`` was used for
instrumentation, use this option to correlate the raw profile.

.. option:: --temporal-profile-trace-reservoir-size

Expand Down Expand Up @@ -346,8 +352,9 @@ OPTIONS
.. option:: --debug-info=<path>

Specify the executable or ``.dSYM`` that contains debug info for the raw profile.
When ``-debug-info-correlate`` was used for instrumentation, use this option
to show the correlated functions from the raw profile.
When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used
for instrumentation, use this option to show the correlated functions from the
raw profile.

.. option:: --covered

Expand Down
4 changes: 2 additions & 2 deletions llvm/include/llvm/ProfileData/InstrProf.h
Original file line number Diff line number Diff line change
Expand Up @@ -328,8 +328,8 @@ enum class instrprof_error {
too_large,
truncated,
malformed,
missing_debug_info_for_correlation,
unexpected_debug_info_for_correlation,
missing_correlation_info,
unexpected_correlation_info,
unable_to_correlate_profile,
unknown_function,
invalid_prof,
Expand Down
55 changes: 43 additions & 12 deletions llvm/include/llvm/ProfileData/InstrProfCorrelator.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// This file defines InstrProfCorrelator used to generate PGO profiles from
// raw profile data and debug info.
// This file defines InstrProfCorrelator used to generate PGO/coverage profiles
// from raw profile data and debug info/binary file.
//===----------------------------------------------------------------------===//

#ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
Expand All @@ -31,8 +31,9 @@ class ObjectFile;
/// to their functions.
class InstrProfCorrelator {
public:
/// Indicate which kind correlator to use.
enum ProfCorrelatorKind { NONE, DEBUG_INFO };
/// Indicate if we should use the debug info or profile metadata sections to
/// correlate.
enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY };

static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(StringRef Filename, ProfCorrelatorKind FileKind);
Expand Down Expand Up @@ -71,11 +72,18 @@ class InstrProfCorrelator {
protected:
struct Context {
static llvm::Expected<std::unique_ptr<Context>>
get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj);
get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj,
ProfCorrelatorKind FileKind);
std::unique_ptr<MemoryBuffer> Buffer;
/// The address range of the __llvm_prf_cnts section.
uint64_t CountersSectionStart;
uint64_t CountersSectionEnd;
/// The pointer points to start/end of profile data/name sections if
/// FileKind is Binary.
const char *DataStart;
const char *DataEnd;
const char *NameStart;
size_t NameSize;
/// True if target and host have different endian orders.
bool ShouldSwapBytes;
};
Expand Down Expand Up @@ -145,19 +153,20 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {

Error dumpYaml(int MaxWarnings, raw_ostream &OS) override;

void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset,
IntPtrT FunctionPtr, uint32_t NumCounters);
void addDataProbe(uint64_t FunctionName, uint64_t CFGHash,
IntPtrT CounterOffset, IntPtrT FunctionPtr,
uint32_t NumCounters);

// Byte-swap the value if necessary.
template <class T> T maybeSwap(T Value) const {
return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value;
}

private:
InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind,
std::unique_ptr<InstrProfCorrelator::Context> Ctx)
: InstrProfCorrelator(Kind, std::move(Ctx)){};
llvm::DenseSet<IntPtrT> CounterOffsets;

// Byte-swap the value if necessary.
template <class T> T maybeSwap(T Value) const {
return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value;
}
};

/// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes
Expand Down Expand Up @@ -214,6 +223,28 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
Error correlateProfileNameImpl() override;
};

/// BinaryInstrProfCorrelator - A child of InstrProfCorrelatorImpl that
/// takes an object file as input to correlate profiles.
template <class IntPtrT>
class BinaryInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
public:
BinaryInstrProfCorrelator(std::unique_ptr<InstrProfCorrelator::Context> Ctx)
: InstrProfCorrelatorImpl<IntPtrT>(std::move(Ctx)) {}

/// Return a pointer to the names string that this class constructs.
const char *getNamesPointer() const { return this->Ctx.NameStart; }

/// Return the number of bytes in the names string.
size_t getNamesSize() const { return this->Ctx.NameSize; }

private:
void correlateProfileDataImpl(
int MaxWarnings,
InstrProfCorrelator::CorrelationData *Data = nullptr) override;

Error correlateProfileNameImpl() override;
};

} // end namespace llvm

#endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
17 changes: 17 additions & 0 deletions llvm/include/llvm/ProfileData/InstrProfData.inc
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_covname, \
INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")

#undef INSTR_PROF_SECT_ENTRY
#endif
Expand Down Expand Up @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
/* Windows section names. Because these section names contain dollar characters,
* they must be quoted.
Expand All @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
/* Since cov data and cov names sections are not allocated, we don't need to
* access them at runtime.
*/
#define INSTR_PROF_COVDATA_COFF ".lcovd"
#define INSTR_PROF_COVNAME_COFF ".lcovn"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"

#ifdef _WIN32
Expand All @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
#else
/* Runtime section names and name strings. */
Expand All @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
/* Order file instrumentation. */
#define INSTR_PROF_ORDERFILE_SECT_NAME \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
Expand Down
9 changes: 0 additions & 9 deletions llvm/include/llvm/ProfileData/InstrProfReader.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,6 @@ class InstrProfReader {

virtual bool instrEntryBBEnabled() const = 0;

/// Return true if we must provide debug info to create PGO profiles.
virtual bool useDebugInfoCorrelate() const { return false; }

/// Return true if the profile has single byte counters representing coverage.
virtual bool hasSingleByteCoverage() const = 0;

Expand Down Expand Up @@ -378,12 +375,6 @@ class RawInstrProfReader : public InstrProfReader {
return (Version & VARIANT_MASK_INSTR_ENTRY) != 0;
}

bool useDebugInfoCorrelate() const override {
return (Version & VARIANT_MASK_DBG_CORRELATE) != 0;
}

bool useCorrelate() const { return useDebugInfoCorrelate(); }

bool hasSingleByteCoverage() const override {
return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0;
}
Expand Down
9 changes: 3 additions & 6 deletions llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2194,20 +2194,17 @@ void AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {

MCSymbol *LazyPointer =
GetExternalSymbolSymbol(GI.getName() + ".lazy_pointer");
MCSymbol *StubHelper =
GetExternalSymbolSymbol(GI.getName() + ".stub_helper");
MCSymbol *StubHelper = GetExternalSymbolSymbol(GI.getName() + ".stub_helper");

OutStreamer->switchSection(
OutContext.getObjectFileInfo()->getDataSection());
OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection());

const DataLayout &DL = M.getDataLayout();
emitAlignment(Align(DL.getPointerSize()));
OutStreamer->emitLabel(LazyPointer);
emitVisibility(LazyPointer, GI.getVisibility());
OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);

OutStreamer->switchSection(
OutContext.getObjectFileInfo()->getTextSection());
OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());

const TargetSubtargetInfo *STI =
TM.getSubtargetImpl(*GI.getResolverFunction());
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,10 @@ static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) {
/*AddSegmentInfo=*/false) ||
Name == getInstrProfSectionName(IPSK_covfun, Triple::ELF,
/*AddSegmentInfo=*/false) ||
Name == getInstrProfSectionName(IPSK_covdata, Triple::ELF,
/*AddSegmentInfo=*/false) ||
Name == getInstrProfSectionName(IPSK_covname, Triple::ELF,
/*AddSegmentInfo=*/false) ||
Name == ".llvmbc" || Name == ".llvmcmd")
return SectionKind::getMetadata();

Expand Down
65 changes: 39 additions & 26 deletions llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,13 @@ Error InstrProfSymtab::create(SectionRef &Section) {

// If this is a linked PE/COFF file, then we have to skip over the null byte
// that is allocated in the .lprfn$A section in the LLVM profiling runtime.
// If the name section is .lprfcovnames, it doesn't have the null byte at the
// beginning.
const ObjectFile *Obj = Section.getObject();
if (isa<COFFObjectFile>(Obj) && !Obj->isRelocatableObject())
Data = Data.drop_front(1);
if (Expected<StringRef> NameOrErr = Section.getName())
if (*NameOrErr != getInstrProfSectionName(IPSK_covname, Triple::COFF))
Data = Data.drop_front(1);

return Error::success();
}
Expand Down Expand Up @@ -1024,10 +1028,13 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
BytesInAddress, Endian, CompilationDir);
}

/// Find all sections that match \p Name. There may be more than one if comdats
/// are in use, e.g. for the __llvm_covfun section on ELF.
static Expected<std::vector<SectionRef>> lookupSections(ObjectFile &OF,
StringRef Name) {
/// Find all sections that match \p IPSK name. There may be more than one if
/// comdats are in use, e.g. for the __llvm_covfun section on ELF.
static Expected<std::vector<SectionRef>>
lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) {
auto ObjFormat = OF.getTripleObjectFormat();
auto Name =
getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false);
// On COFF, the object file section name may end in "$M". This tells the
// linker to sort these sections between "$A" and "$Z". The linker removes the
// dollar and everything after it in the final binary. Do the same to match.
Expand All @@ -1042,8 +1049,13 @@ static Expected<std::vector<SectionRef>> lookupSections(ObjectFile &OF,
Expected<StringRef> NameOrErr = Section.getName();
if (!NameOrErr)
return NameOrErr.takeError();
if (stripSuffix(*NameOrErr) == Name)
if (stripSuffix(*NameOrErr) == Name) {
// COFF profile name section contains two null bytes indicating the
// start/end of the section. If its size is 2 bytes, it's empty.
if (IsCOFF && IPSK == IPSK_name && Section.getSize() == 2)
continue;
Sections.push_back(Section);
}
}
if (Sections.empty())
return make_error<CoverageMapError>(coveragemap_error::no_data_found);
Expand Down Expand Up @@ -1079,15 +1091,27 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
OF->isLittleEndian() ? llvm::endianness::little : llvm::endianness::big;

// Look for the sections that we are interested in.
auto ObjFormat = OF->getTripleObjectFormat();
auto NamesSection =
lookupSections(*OF, getInstrProfSectionName(IPSK_name, ObjFormat,
/*AddSegmentInfo=*/false));
if (auto E = NamesSection.takeError())
InstrProfSymtab ProfileNames;
std::vector<SectionRef> NamesSectionRefs;
// If IPSK_name is not found, fallback to search for IPK_covname, which is
// used when binary correlation is enabled.
auto NamesSection = lookupSections(*OF, IPSK_name);
if (auto E = NamesSection.takeError()) {
consumeError(std::move(E));
NamesSection = lookupSections(*OF, IPSK_covname);
if (auto E = NamesSection.takeError())
return std::move(E);
}
NamesSectionRefs = *NamesSection;

if (NamesSectionRefs.size() != 1)
return make_error<CoverageMapError>(
coveragemap_error::malformed,
"the size of coverage mapping section is not one");
if (Error E = ProfileNames.create(NamesSectionRefs.back()))
return std::move(E);
auto CoverageSection =
lookupSections(*OF, getInstrProfSectionName(IPSK_covmap, ObjFormat,
/*AddSegmentInfo=*/false));

auto CoverageSection = lookupSections(*OF, IPSK_covmap);
if (auto E = CoverageSection.takeError())
return std::move(E);
std::vector<SectionRef> CoverageSectionRefs = *CoverageSection;
Expand All @@ -1099,19 +1123,8 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
return CoverageMappingOrErr.takeError();
StringRef CoverageMapping = CoverageMappingOrErr.get();

InstrProfSymtab ProfileNames;
std::vector<SectionRef> NamesSectionRefs = *NamesSection;
if (NamesSectionRefs.size() != 1)
return make_error<CoverageMapError>(
coveragemap_error::malformed,
"the size of coverage mapping section is not one");
if (Error E = ProfileNames.create(NamesSectionRefs.back()))
return std::move(E);

// Look for the coverage records section (Version4 only).
auto CoverageRecordsSections =
lookupSections(*OF, getInstrProfSectionName(IPSK_covfun, ObjFormat,
/*AddSegmentInfo=*/false));
auto CoverageRecordsSections = lookupSections(*OF, IPSK_covfun);

BinaryCoverageReader::FuncRecordsStorage FuncRecords;
if (auto E = CoverageRecordsSections.takeError()) {
Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/ProfileData/InstrProf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,11 @@ static std::string getInstrProfErrString(instrprof_error Err,
case instrprof_error::malformed:
OS << "malformed instrumentation profile data";
break;
case instrprof_error::missing_debug_info_for_correlation:
OS << "debug info for correlation is required";
case instrprof_error::missing_correlation_info:
OS << "debug info/binary for correlation is required";
break;
case instrprof_error::unexpected_debug_info_for_correlation:
OS << "debug info for correlation is not necessary";
case instrprof_error::unexpected_correlation_info:
OS << "debug info/binary for correlation is not necessary";
break;
case instrprof_error::unable_to_correlate_profile:
OS << "unable to correlate profile";
Expand Down
119 changes: 103 additions & 16 deletions llvm/lib/ProfileData/InstrProfCorrelator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,22 @@ using namespace llvm;
/// Get profile section.
Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj,
InstrProfSectKind IPSK) {
// On COFF, the getInstrProfSectionName returns the section names may followed
// by "$M". The linker removes the dollar and everything after it in the final
// binary. Do the same to match.
Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
auto StripSuffix = [ObjFormat](StringRef N) {
return ObjFormat == Triple::COFF ? N.split('$').first : N;
};
std::string ExpectedSectionName =
getInstrProfSectionName(IPSK, ObjFormat,
/*AddSegmentInfo=*/false);
for (auto &Section : Obj.sections())
ExpectedSectionName = StripSuffix(ExpectedSectionName);
for (auto &Section : Obj.sections()) {
if (auto SectionName = Section.getName())
if (SectionName.get() == ExpectedSectionName)
if (*SectionName == ExpectedSectionName)
return Section;
}
return make_error<InstrProfError>(
instrprof_error::unable_to_correlate_profile,
"could not find section (" + Twine(ExpectedSectionName) + ")");
Expand All @@ -46,14 +54,38 @@ const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";

llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
const object::ObjectFile &Obj) {
const object::ObjectFile &Obj,
ProfCorrelatorKind FileKind) {
auto C = std::make_unique<Context>();
auto CountersSection = getInstrProfSection(Obj, IPSK_cnts);
if (auto Err = CountersSection.takeError())
return std::move(Err);
auto C = std::make_unique<Context>();
if (FileKind == InstrProfCorrelator::BINARY) {
auto DataSection = getInstrProfSection(Obj, IPSK_covdata);
if (auto Err = DataSection.takeError())
return std::move(Err);
auto DataOrErr = DataSection->getContents();
if (!DataOrErr)
return DataOrErr.takeError();
auto NameSection = getInstrProfSection(Obj, IPSK_covname);
if (auto Err = NameSection.takeError())
return std::move(Err);
auto NameOrErr = NameSection->getContents();
if (!NameOrErr)
return NameOrErr.takeError();
C->DataStart = DataOrErr->data();
C->DataEnd = DataOrErr->data() + DataOrErr->size();
C->NameStart = NameOrErr->data();
C->NameSize = NameOrErr->size();
}
C->Buffer = std::move(Buffer);
C->CountersSectionStart = CountersSection->getAddress();
C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
// In COFF object file, there's a null byte at the beginning of the counter
// section which doesn't exist in raw profile.
if (Obj.getTripleObjectFormat() == Triple::COFF)
++C->CountersSectionStart;

C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
return Expected<std::unique_ptr<Context>>(std::move(C));
}
Expand All @@ -80,9 +112,17 @@ InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) {

return get(std::move(*BufferOrErr), FileKind);
}
if (FileKind == BINARY) {
auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));
if (auto Err = BufferOrErr.takeError())
return std::move(Err);

return get(std::move(*BufferOrErr), FileKind);
}
return make_error<InstrProfError>(
instrprof_error::unable_to_correlate_profile,
"unsupported correlation kind (only DWARF debug info is supported)");
"unsupported correlation kind (only DWARF debug info and Binary format "
"(ELF/COFF) are supported)");
}

llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
Expand All @@ -93,7 +133,7 @@ InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,
return std::move(Err);

if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind);
if (auto Err = CtxOrErr.takeError())
return std::move(Err);
auto T = Obj->makeTriple();
Expand Down Expand Up @@ -155,9 +195,11 @@ InstrProfCorrelatorImpl<IntPtrT>::get(
instrprof_error::unable_to_correlate_profile,
"unsupported debug info format (only DWARF is supported)");
}
if (Obj.isELF() || Obj.isCOFF())
return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx));
return make_error<InstrProfError>(
instrprof_error::unable_to_correlate_profile,
"unsupported correlation file type (only DWARF is supported)");
"unsupported binary format (only ELF and COFF are supported)");
}

template <class IntPtrT>
Expand Down Expand Up @@ -212,16 +254,16 @@ Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
}

template <class IntPtrT>
void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
uint64_t CFGHash,
IntPtrT CounterOffset,
IntPtrT FunctionPtr,
uint32_t NumCounters) {
void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
uint64_t CFGHash,
IntPtrT CounterOffset,
IntPtrT FunctionPtr,
uint32_t NumCounters) {
// Check if a probe was already added for this counter offset.
if (!CounterOffsets.insert(CounterOffset).second)
return;
Data.push_back({
maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
maybeSwap<uint64_t>(NameRef),
maybeSwap<uint64_t>(CFGHash),
// In this mode, CounterPtr actually stores the section relative address
// of the counter.
Expand All @@ -236,7 +278,6 @@ void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
// TODO: MC/DC is not yet supported.
/*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
});
NamesVec.push_back(FunctionName.str());
}

template <class IntPtrT>
Expand Down Expand Up @@ -349,6 +390,8 @@ void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
*FunctionName);
LLVM_DEBUG(Die.dump(dbgs()));
}
// In debug info correlation mode, the CounterPtr is an absolute address of
// the counter, but it's expected to be relative later when iterating Data.
IntPtrT CounterOffset = *CounterPtr - CountersStart;
if (Data) {
InstrProfCorrelator::Probe P;
Expand All @@ -366,8 +409,9 @@ void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
P.LineNumber = LineNumber;
Data->Probes.push_back(P);
} else {
this->addProbe(*FunctionName, *CFGHash, CounterOffset,
FunctionPtr.value_or(0), *NumCounters);
this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash,
CounterOffset, FunctionPtr.value_or(0), *NumCounters);
this->NamesVec.push_back(*FunctionName);
}
};
for (auto &CU : DICtx->normal_units())
Expand All @@ -394,3 +438,46 @@ Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
/*doCompression=*/false, this->Names);
return Result;
}

template <class IntPtrT>
void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) {
using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
bool UnlimitedWarnings = (MaxWarnings == 0);
// -N suppressed warnings means we can emit up to N (unsuppressed) warnings
int NumSuppressedWarnings = -MaxWarnings;

const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart;
const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd;
// We need to use < here because the last data record may have no padding.
for (const RawProfData *I = DataStart; I < DataEnd; ++I) {
uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr);
uint64_t CountersStart = this->Ctx->CountersSectionStart;
uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) {
if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
WithColor::warning()
<< format("CounterPtr out of range for function: Actual=0x%x "
"Expected=[0x%x, 0x%x) at data offset=0x%x\n",
CounterPtr, CountersStart, CountersEnd,
(I - DataStart) * sizeof(RawProfData));
}
}
// In binary correlation mode, the CounterPtr is an absolute address of the
// counter, but it's expected to be relative later when iterating Data.
IntPtrT CounterOffset = CounterPtr - CountersStart;
this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,
I->FunctionPointer, I->NumCounters);
}
}

template <class IntPtrT>
Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
if (this->Ctx->NameSize == 0) {
return make_error<InstrProfError>(
instrprof_error::unable_to_correlate_profile,
"could not find any profile data metadata in object file");
}
this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
return Error::success();
}
9 changes: 3 additions & 6 deletions llvm/lib/ProfileData/InstrProfReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -556,10 +556,6 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
"\nPLEASE update this tool to version in the raw profile, or "
"regenerate raw profile with expected version.")
.str());
if (useCorrelate() && !Correlator)
return error(instrprof_error::missing_debug_info_for_correlation);
if (!useCorrelate() && Correlator)
return error(instrprof_error::unexpected_debug_info_for_correlation);

uint64_t BinaryIdSize = swap(Header.BinaryIdsSize);
// Binary id start just after the header if exists.
Expand Down Expand Up @@ -607,8 +603,9 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
if (Correlator) {
// These sizes in the raw file are zero because we constructed them in the
// Correlator.
assert(DataSize == 0 && NamesSize == 0);
assert(CountersDelta == 0 && NamesDelta == 0);
if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
NamesDelta == 0))
return error(instrprof_error::unexpected_correlation_info);
Data = Correlator->getDataPointer();
DataEnd = Data + Correlator->getDataSize();
NamesStart = Correlator->getNamesPointer();
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ void PPCInstPrinter::printAbsBranchOperand(const MCInst *MI, unsigned OpNo,
if (!MI->getOperand(OpNo).isImm())
return printOperand(MI, OpNo, STI, O);

uint64_t Imm = MI->getOperand(OpNo).getImm() << 2;
uint64_t Imm = static_cast<uint64_t>(MI->getOperand(OpNo).getImm()) << 2;
if (!TT.isPPC64())
Imm = static_cast<uint32_t>(Imm);
O << formatHex(Imm);
Expand Down
71 changes: 49 additions & 22 deletions llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,24 @@ using namespace llvm;
#define DEBUG_TYPE "instrprof"

namespace llvm {
cl::opt<bool>
DebugInfoCorrelate("debug-info-correlate",
cl::desc("Use debug info to correlate profiles."),
cl::init(false));
// TODO: Remove -debug-info-correlate in next LLVM release, in favor of
// -profile-correlate=debug-info.
cl::opt<bool> DebugInfoCorrelate(
"debug-info-correlate",
cl::desc("Use debug info to correlate profiles. (Deprecated, use "
"-profile-correlate=debug-info)"),
cl::init(false));

cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
"profile-correlate",
cl::desc("Use debug info or binary file to correlate profiles."),
cl::init(InstrProfCorrelator::NONE),
cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
"No profile correlation"),
clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
"Use debug info to correlate"),
clEnumValN(InstrProfCorrelator::BINARY, "binary",
"Use binary to correlate")));
} // namespace llvm

namespace {
Expand Down Expand Up @@ -792,7 +806,7 @@ void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
// in lightweight mode. We need to move the value profile pointer to the
// Counter struct to get this working.
assert(
!DebugInfoCorrelate &&
!DebugInfoCorrelate && ProfileCorrelate == InstrProfCorrelator::NONE &&
"Value profiling is not yet supported with lightweight instrumentation");
GlobalVariable *Name = Ind->getName();
auto It = ProfileDataMap.find(Name);
Expand Down Expand Up @@ -1219,8 +1233,9 @@ GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,

// Use internal rather than private linkage so the counter variable shows up
// in the symbol table when using debug info for correlation.
if (DebugInfoCorrelate && TT.isOSBinFormatMachO() &&
Linkage == GlobalValue::PrivateLinkage)
if ((DebugInfoCorrelate ||
ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) &&
TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
Linkage = GlobalValue::InternalLinkage;

// Due to the limitation of binder as of 2021/09/28, the duplicate weak
Expand Down Expand Up @@ -1341,7 +1356,8 @@ InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
PD.RegionCounters = CounterPtr;

if (DebugInfoCorrelate) {
if (DebugInfoCorrelate ||
ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
LLVMContext &Ctx = M.getContext();
Function *Fn = Inc->getParent()->getParent();
if (auto *SP = Fn->getSubprogram()) {
Expand Down Expand Up @@ -1386,7 +1402,7 @@ InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
// When debug information is correlated to profile data, a data variable
// is not needed.
if (DebugInfoCorrelate)
if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
return;

GlobalVariable *NamePtr = Inc->getName();
Expand Down Expand Up @@ -1484,20 +1500,28 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
}
auto *Data =
new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
// Reference the counter variable with a label difference (link-time
// constant).
auto *RelativeCounterPtr =
ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
ConstantExpr::getPtrToInt(Data, IntPtrTy));

// Bitmaps are relative to the same data variable as profile counters.
Constant *RelativeCounterPtr;
GlobalVariable *BitmapPtr = PD.RegionBitmaps;
Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);

if (BitmapPtr != nullptr) {
RelativeBitmapPtr =
ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy),
InstrProfSectKind DataSectionKind;
// With binary profile correlation, profile data is not loaded into memory.
// profile data must reference profile counter with an absolute relocation.
if (ProfileCorrelate == InstrProfCorrelator::BINARY) {
DataSectionKind = IPSK_covdata;
RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy);
if (BitmapPtr != nullptr)
RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy);
} else {
// Reference the counter variable with a label difference (link-time
// constant).
DataSectionKind = IPSK_data;
RelativeCounterPtr =
ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
ConstantExpr::getPtrToInt(Data, IntPtrTy));
if (BitmapPtr != nullptr)
RelativeBitmapPtr =
ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy),
ConstantExpr::getPtrToInt(Data, IntPtrTy));
}

Constant *DataVals[] = {
Expand All @@ -1507,7 +1531,8 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
Data->setInitializer(ConstantStruct::get(DataTy, DataVals));

Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
Data->setSection(
getInstrProfSectionName(DataSectionKind, TT.getObjectFormat()));
Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
maybeSetComdat(Data, Fn, CntsVarName);

Expand Down Expand Up @@ -1595,7 +1620,9 @@ void InstrLowerer::emitNameData() {
NamesSize = CompressedNameStr.size();
setGlobalVariableLargeSection(TT, *NamesVar);
NamesVar->setSection(
getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
ProfileCorrelate == InstrProfCorrelator::BINARY
? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat())
: getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
// On COFF, it's important to reduce the alignment down to 1 to prevent the
// linker from inserting padding before the start of the names section or
// between names entries.
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,7 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;

extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
} // namespace llvm

static cl::opt<bool>
Expand Down Expand Up @@ -381,7 +382,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) {
ProfileVersion |= VARIANT_MASK_CSIR_PROF;
if (PGOInstrumentEntry)
ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
if (DebugInfoCorrelate)
if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
if (PGOFunctionEntryCoverage)
ProfileVersion |=
Expand Down
8 changes: 8 additions & 0 deletions llvm/test/Instrumentation/InstrProfiling/coverage.ll
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
; RUN: opt < %s -passes=instrprof -S | FileCheck %s
; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s --check-prefix=BINARY

target triple = "aarch64-unknown-linux-gnu"

@__profn_foo = private constant [3 x i8] c"foo"
; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64),
@__profn_bar = private constant [3 x i8] c"bar"
; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1
; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64),

; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names"
; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames"

define void @_Z3foov() {
call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 1, i32 0)
Expand Down
29 changes: 24 additions & 5 deletions llvm/tools/llvm-profdata/llvm-profdata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
#include <queue>

using namespace llvm;
using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind;

// https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations
// on each subcommand.
Expand Down Expand Up @@ -124,6 +125,11 @@ cl::opt<std::string> DebugInfoFilename(
"the functions it found. For merge, use the provided debug info to "
"correlate the raw profile."),
cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
cl::opt<std::string>
BinaryFilename("binary-file", cl::init(""),
cl::desc("For merge, use the provided unstripped bianry to "
"correlate the raw profile."),
cl::sub(MergeSubcommand));
cl::opt<std::string> FuncNameFilter(
"function",
cl::desc("Details for matching functions. For overlapping CSSPGO, this "
Expand Down Expand Up @@ -787,14 +793,27 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs,
OutputFormat != PF_Text)
exitWithError("unknown format is specified");

std::unique_ptr<InstrProfCorrelator> Correlator;
// TODO: Maybe we should support correlation with mixture of different
// correlation modes(w/wo debug-info/object correlation).
if (!DebugInfoFilename.empty() && !BinaryFilename.empty())
exitWithError("Expected only one of -debug-info, -binary-file");
std::string CorrelateFilename;
ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE;
if (!DebugInfoFilename.empty()) {
if (auto Err = InstrProfCorrelator::get(DebugInfoFilename,
InstrProfCorrelator::DEBUG_INFO)
CorrelateFilename = DebugInfoFilename;
CorrelateKind = ProfCorrelatorKind::DEBUG_INFO;
} else if (!BinaryFilename.empty()) {
CorrelateFilename = BinaryFilename;
CorrelateKind = ProfCorrelatorKind::BINARY;
}

std::unique_ptr<InstrProfCorrelator> Correlator;
if (CorrelateKind != InstrProfCorrelator::NONE) {
if (auto Err = InstrProfCorrelator::get(CorrelateFilename, CorrelateKind)
.moveInto(Correlator))
exitWithError(std::move(Err), DebugInfoFilename);
exitWithError(std::move(Err), CorrelateFilename);
if (auto Err = Correlator->correlateProfileData(MaxDbgCorrelationWarnings))
exitWithError(std::move(Err), DebugInfoFilename);
exitWithError(std::move(Err), CorrelateFilename);
}

std::mutex ErrorLock;
Expand Down
3 changes: 3 additions & 0 deletions mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ LogicalResult moduleSerializer(GPUModuleOp op,
const TargetOptions &targetOptions) {
OpBuilder builder(op->getContext());
SmallVector<Attribute> objects;
// Fail if there are no target attributes
if (!op.getTargetsAttr())
return op.emitError("the module has no target attributes");
// Serialize all targets.
for (auto targetAttr : op.getTargetsAttr()) {
assert(targetAttr && "Target attribute cannot be null.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,23 @@ class AffineDimFinder : public AffineExprVisitor<AffineDimFinder> {
explicit AffineDimFinder(ArrayRef<utils::IteratorType> itTypes)
: iterTypes(itTypes) {}

// Override method from AffineExprVisitor.
/// Overrides the visit method from AffineExprVisitor.
void visitDimExpr(AffineDimExpr expr) {
if (pickedDim == nullptr || pickIterType == iterTypes[expr.getPosition()])
pickedDim = expr;
}

/// Set the desired iterator type that we want to pick.
/// Sets the desired iterator type that we want to pick.
void setPickedIterType(utils::IteratorType iterType) {
pickIterType = iterType;
}

/// Get the desired AffineDimExpr.
/// Gets the desired AffineDimExpr.
AffineDimExpr getDimExpr() const {
return llvm::cast<AffineDimExpr>(pickedDim);
}

/// Walks the graph in post order to find dim expr.
void walkPostOrder(AffineExpr expr) {
pickedDim = nullptr;
AffineExprVisitor<AffineDimFinder>::walkPostOrder(expr);
Expand All @@ -55,11 +56,11 @@ class AffineDimFinder : public AffineExprVisitor<AffineDimFinder> {
AffineExpr pickedDim;
/// The iterator type that we want.
utils::IteratorType pickIterType;
/// The mapping between dim=>iterator type.
/// The mapping between levels and iterator types.
ArrayRef<utils::IteratorType> iterTypes;
};

// Flattens an affine expression into a list of AffineDimExprs.
/// Flattens an affine expression into a list of AffineDimExprs.
struct AffineDimCollector : public AffineExprVisitor<AffineDimCollector> {
// Overrides method from AffineExprVisitor.
void visitDimExpr(AffineDimExpr expr) { dims.push_back(expr); }
Expand Down Expand Up @@ -97,8 +98,8 @@ AffineMap IterationGraphSorter::topoSort() {

SmallVector<unsigned> loopOrder;
while (!redIt.empty() || !parIt.empty()) {
// We always prefer parallel loop over reduction loop because putting
// reduction loop early might make the loop sequence inadmissible.
// We always prefer a parallel loop over a reduction loop because putting
// a reduction loop early might make the loop sequence inadmissible.
auto &it = !parIt.empty() ? parIt : redIt;
auto src = it.back();
loopOrder.push_back(src);
Expand All @@ -114,6 +115,7 @@ AffineMap IterationGraphSorter::topoSort() {
}
}

// Return the topological sort on success.
if (loopOrder.size() == numLoops)
return AffineMap::getPermutationMap(loopOrder, out.getContext());

Expand Down Expand Up @@ -164,13 +166,14 @@ IterationGraphSorter::IterationGraphSorter(
}

AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
// Reset the interation graph.
// Reset the adjacency matrix that represents the iteration graph.
for (auto &row : itGraph)
std::fill(row.begin(), row.end(), false);

// Reset cached in-degree.
// Reset in-degree.
std::fill(inDegree.begin(), inDegree.end(), 0);

// Add the constraints for the loop to level map.
for (auto [in, map] : llvm::zip(ins, loop2InsLvl)) {
// Get map and encoding.
const auto enc = getSparseTensorEncoding(in.getType());
Expand All @@ -180,11 +183,12 @@ AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
addConstraints(in, map);
}

// Get map and encoding.
// Add the constraints for the output map.
const auto enc = getSparseTensorEncoding(out.getType());
if ((enc || includesDenseOutput(mask)) && out != ignored)
addConstraints(out, loop2OutLvl);

// Return the topological sort (empty for cyclic).
return topoSort();
}

Expand All @@ -196,6 +200,7 @@ void IterationGraphSorter::addConstraints(Value t, AffineMap loop2LvlMap) {
}
};

// Set up a reduction finder.
AffineDimFinder finder(iterTypes);
finder.setPickedIterType(utils::IteratorType::reduction);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class IterationGraphSorter {
// Loop itation types;
SmallVector<utils::IteratorType> iterTypes;

// Adjacent matrix that represents the iteration graph.
// Adjacency matrix that represents the iteration graph.
std::vector<std::vector<bool>> itGraph;

// InDegree used for topo sort.
Expand Down
12 changes: 12 additions & 0 deletions mlir/test/Dialect/GPU/module-to-binary-invalid.mlir
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// RUN: mlir-opt %s --gpu-module-to-binary --verify-diagnostics

module attributes {gpu.container_module} {
// expected-error @below {{the module has no target attributes}}
gpu.module @kernel_module1 {
llvm.func @kernel(%arg0: i32, %arg1: !llvm.ptr,
%arg2: !llvm.ptr, %arg3: i64, %arg4: i64,
%arg5: i64) attributes {gpu.kernel} {
llvm.return
}
}
}