diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 77455c075cab0..7d16de33763a0 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -45,6 +45,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" @@ -102,17 +103,21 @@ static cl::opt ClSanitizeOnOptimizerEarlyEP( "sanitizer-early-opt-ep", cl::Optional, cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); +extern cl::opt ProfileCorrelate; + // Re-link builtin bitcodes after optimization cl::opt ClRelinkBuiltinBitcodePostop( "relink-builtin-bitcode-postop", cl::Optional, cl::desc("Re-link builtin bitcodes after optimization."), cl::init(false)); -} +} // namespace llvm namespace { // Default filename used for profile generation. std::string getDefaultProfileGenName() { - return DebugInfoCorrelate ? "default_%m.proflite" : "default_%m.profraw"; + return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE + ? "default_%m.proflite" + : "default_%m.profraw"; } class EmitAssemblyHelper { @@ -204,7 +209,7 @@ class EmitAssemblyHelper { void EmitAssembly(BackendAction Action, std::unique_ptr OS, BackendConsumer *BC); }; -} +} // namespace static SanitizerCoverageOptions getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) { diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 44a449800923f..f5de23ff4b94d 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9dbd702865fd2..9070b8a606eb5 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,13 +13,14 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") #pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. */ /* Allocate read-only section bounds. */ diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index f9b01b15b0e62..7357604b1f651 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -37,8 +37,9 @@ if(NOT ANDROID) if(NOT COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD) # Use LLVM utils and Clang from the same build tree. list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS - clang clang-resource-headers FileCheck count not llvm-config llvm-nm llvm-objdump - llvm-readelf llvm-readobj llvm-size llvm-symbolizer compiler-rt-headers sancov split-file) + clang clang-resource-headers FileCheck count not llvm-config llvm-nm + llvm-objdump llvm-readelf llvm-readobj llvm-size llvm-symbolizer + compiler-rt-headers sancov split-file llvm-strip) if (WIN32) list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS KillTheDoctor) endif() diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c new file mode 100644 index 0000000000000..8f421014cf5c9 --- /dev/null +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -0,0 +1,46 @@ +// REQUIRES: linux || windows +// Default +// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report +// RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show + +// With -profile-correlate=binary flag +// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe +// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw +// RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report +// RUN: llvm-cov show --instr-profile=%t-1.profdata %t-1.exe > %t-1.show +// RUN: diff %t.normal.profdata %t-1.profdata +// RUN: diff %t.normal.report %t-1.report +// RUN: diff %t.normal.show %t-1.show + +// Strip above binary and run +// RUN: llvm-strip %t-1.exe -o %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw +// RUN: llvm-cov report --instr-profile=%t-2.profdata %t-1.exe > %t-2.report +// RUN: llvm-cov show --instr-profile=%t-2.profdata %t-1.exe > %t-2.show +// RUN: diff %t.normal.profdata %t-2.profdata +// RUN: diff %t.normal.report %t-2.report +// RUN: diff %t.normal.show %t-2.show + +// Online merging. +// RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal +// RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.report +// RUN: llvm-cov show --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.show + +// RUN: rm -rf %t.profdir && mkdir %t.profdir +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir +// RUN: llvm-cov report --instr-profile=%t-4.profdata %t-1.exe > %t-4.report +// RUN: llvm-cov show --instr-profile=%t-4.profdata %t-1.exe > %t-4.show +// RUN: diff %t.normal.merged.profdata %t-4.profdata +// RUN: diff %t.normal.merged.report %t-4.report +// RUN: diff %t.normal.merged.show %t-4.show + +// TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst index be42733ca1405..f5e3c13ffbc8e 100644 --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -195,8 +195,14 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to correlate the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to correlate the raw profile. + +.. option:: --binary-file= + + Specify the executable that contains profile data and profile name sections for + the raw profile. When ``-profile-correlate=binary`` was used for + instrumentation, use this option to correlate the raw profile. .. option:: --temporal-profile-trace-reservoir-size @@ -346,8 +352,9 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to show the correlated functions from the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to show the correlated functions from the + raw profile. .. option:: --covered diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 3bc677d5b6d86..288dc71d756ae 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -328,8 +328,8 @@ enum class instrprof_error { too_large, truncated, malformed, - missing_debug_info_for_correlation, - unexpected_debug_info_for_correlation, + missing_correlation_info, + unexpected_correlation_info, unable_to_correlate_profile, unknown_function, invalid_prof, diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index a3a0805a294a2..c07c67d287e2c 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -5,8 +5,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This file defines InstrProfCorrelator used to generate PGO profiles from -// raw profile data and debug info. +// This file defines InstrProfCorrelator used to generate PGO/coverage profiles +// from raw profile data and debug info/binary file. //===----------------------------------------------------------------------===// #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H @@ -31,8 +31,9 @@ class ObjectFile; /// to their functions. class InstrProfCorrelator { public: - /// Indicate which kind correlator to use. - enum ProfCorrelatorKind { NONE, DEBUG_INFO }; + /// Indicate if we should use the debug info or profile metadata sections to + /// correlate. + enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY }; static llvm::Expected> get(StringRef Filename, ProfCorrelatorKind FileKind); @@ -71,11 +72,18 @@ class InstrProfCorrelator { protected: struct Context { static llvm::Expected> - get(std::unique_ptr Buffer, const object::ObjectFile &Obj); + get(std::unique_ptr Buffer, const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind); std::unique_ptr Buffer; /// The address range of the __llvm_prf_cnts section. uint64_t CountersSectionStart; uint64_t CountersSectionEnd; + /// The pointer points to start/end of profile data/name sections if + /// FileKind is Binary. + const char *DataStart; + const char *DataEnd; + const char *NameStart; + size_t NameSize; /// True if target and host have different endian orders. bool ShouldSwapBytes; }; @@ -145,19 +153,20 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator { Error dumpYaml(int MaxWarnings, raw_ostream &OS) override; - void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, - IntPtrT FunctionPtr, uint32_t NumCounters); + void addDataProbe(uint64_t FunctionName, uint64_t CFGHash, + IntPtrT CounterOffset, IntPtrT FunctionPtr, + uint32_t NumCounters); + + // Byte-swap the value if necessary. + template T maybeSwap(T Value) const { + return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; + } private: InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind, std::unique_ptr Ctx) : InstrProfCorrelator(Kind, std::move(Ctx)){}; llvm::DenseSet CounterOffsets; - - // Byte-swap the value if necessary. - template T maybeSwap(T Value) const { - return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; - } }; /// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes @@ -214,6 +223,28 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl { Error correlateProfileNameImpl() override; }; +/// BinaryInstrProfCorrelator - A child of InstrProfCorrelatorImpl that +/// takes an object file as input to correlate profiles. +template +class BinaryInstrProfCorrelator : public InstrProfCorrelatorImpl { +public: + BinaryInstrProfCorrelator(std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(std::move(Ctx)) {} + + /// Return a pointer to the names string that this class constructs. + const char *getNamesPointer() const { return this->Ctx.NameStart; } + + /// Return the number of bytes in the names string. + size_t getNamesSize() const { return this->Ctx.NameSize; } + +private: + void correlateProfileDataImpl( + int MaxWarnings, + InstrProfCorrelator::CorrelationData *Data = nullptr) override; + + Error correlateProfileNameImpl() override; +}; + } // end namespace llvm #endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 44a449800923f..f5de23ff4b94d 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -701,6 +707,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. @@ -713,6 +721,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -729,6 +742,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ @@ -744,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 952cc0d0dc80b..ff50dfde0e793 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -123,9 +123,6 @@ class InstrProfReader { virtual bool instrEntryBBEnabled() const = 0; - /// Return true if we must provide debug info to create PGO profiles. - virtual bool useDebugInfoCorrelate() const { return false; } - /// Return true if the profile has single byte counters representing coverage. virtual bool hasSingleByteCoverage() const = 0; @@ -378,12 +375,6 @@ class RawInstrProfReader : public InstrProfReader { return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; } - bool useDebugInfoCorrelate() const override { - return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; - } - - bool useCorrelate() const { return useDebugInfoCorrelate(); } - bool hasSingleByteCoverage() const override { return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; } diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 18f4d546aa82d..9a0dd92bb58e8 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -472,6 +472,10 @@ static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) { /*AddSegmentInfo=*/false) || Name == getInstrProfSectionName(IPSK_covfun, Triple::ELF, /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(IPSK_covdata, Triple::ELF, + /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(IPSK_covname, Triple::ELF, + /*AddSegmentInfo=*/false) || Name == ".llvmbc" || Name == ".llvmcmd") return SectionKind::getMetadata(); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 56a7ab24a2f08..ac8e6b56379f2 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -493,9 +493,13 @@ Error InstrProfSymtab::create(SectionRef &Section) { // If this is a linked PE/COFF file, then we have to skip over the null byte // that is allocated in the .lprfn$A section in the LLVM profiling runtime. + // If the name section is .lprfcovnames, it doesn't have the null byte at the + // beginning. const ObjectFile *Obj = Section.getObject(); if (isa(Obj) && !Obj->isRelocatableObject()) - Data = Data.drop_front(1); + if (Expected NameOrErr = Section.getName()) + if (*NameOrErr != getInstrProfSectionName(IPSK_covname, Triple::COFF)) + Data = Data.drop_front(1); return Error::success(); } @@ -1024,10 +1028,13 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) { BytesInAddress, Endian, CompilationDir); } -/// Find all sections that match \p Name. There may be more than one if comdats -/// are in use, e.g. for the __llvm_covfun section on ELF. -static Expected> lookupSections(ObjectFile &OF, - StringRef Name) { +/// Find all sections that match \p IPSK name. There may be more than one if +/// comdats are in use, e.g. for the __llvm_covfun section on ELF. +static Expected> +lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) { + auto ObjFormat = OF.getTripleObjectFormat(); + auto Name = + getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); // On COFF, the object file section name may end in "$M". This tells the // linker to sort these sections between "$A" and "$Z". The linker removes the // dollar and everything after it in the final binary. Do the same to match. @@ -1042,8 +1049,13 @@ static Expected> lookupSections(ObjectFile &OF, Expected NameOrErr = Section.getName(); if (!NameOrErr) return NameOrErr.takeError(); - if (stripSuffix(*NameOrErr) == Name) + if (stripSuffix(*NameOrErr) == Name) { + // COFF profile name section contains two null bytes indicating the + // start/end of the section. If its size is 2 bytes, it's empty. + if (IsCOFF && IPSK == IPSK_name && Section.getSize() == 2) + continue; Sections.push_back(Section); + } } if (Sections.empty()) return make_error(coveragemap_error::no_data_found); @@ -1079,15 +1091,27 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, OF->isLittleEndian() ? llvm::endianness::little : llvm::endianness::big; // Look for the sections that we are interested in. - auto ObjFormat = OF->getTripleObjectFormat(); - auto NamesSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_name, ObjFormat, - /*AddSegmentInfo=*/false)); - if (auto E = NamesSection.takeError()) + InstrProfSymtab ProfileNames; + std::vector NamesSectionRefs; + // If IPSK_name is not found, fallback to search for IPK_covname, which is + // used when binary correlation is enabled. + auto NamesSection = lookupSections(*OF, IPSK_name); + if (auto E = NamesSection.takeError()) { + consumeError(std::move(E)); + NamesSection = lookupSections(*OF, IPSK_covname); + if (auto E = NamesSection.takeError()) + return std::move(E); + } + NamesSectionRefs = *NamesSection; + + if (NamesSectionRefs.size() != 1) + return make_error( + coveragemap_error::malformed, + "the size of coverage mapping section is not one"); + if (Error E = ProfileNames.create(NamesSectionRefs.back())) return std::move(E); - auto CoverageSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_covmap, ObjFormat, - /*AddSegmentInfo=*/false)); + + auto CoverageSection = lookupSections(*OF, IPSK_covmap); if (auto E = CoverageSection.takeError()) return std::move(E); std::vector CoverageSectionRefs = *CoverageSection; @@ -1099,19 +1123,8 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, return CoverageMappingOrErr.takeError(); StringRef CoverageMapping = CoverageMappingOrErr.get(); - InstrProfSymtab ProfileNames; - std::vector NamesSectionRefs = *NamesSection; - if (NamesSectionRefs.size() != 1) - return make_error( - coveragemap_error::malformed, - "the size of coverage mapping section is not one"); - if (Error E = ProfileNames.create(NamesSectionRefs.back())) - return std::move(E); - // Look for the coverage records section (Version4 only). - auto CoverageRecordsSections = - lookupSections(*OF, getInstrProfSectionName(IPSK_covfun, ObjFormat, - /*AddSegmentInfo=*/false)); + auto CoverageRecordsSections = lookupSections(*OF, IPSK_covfun); BinaryCoverageReader::FuncRecordsStorage FuncRecords; if (auto E = CoverageRecordsSections.takeError()) { diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index a53e44d85d090..649d814cfd9de 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -113,11 +113,11 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::malformed: OS << "malformed instrumentation profile data"; break; - case instrprof_error::missing_debug_info_for_correlation: - OS << "debug info for correlation is required"; + case instrprof_error::missing_correlation_info: + OS << "debug info/binary for correlation is required"; break; - case instrprof_error::unexpected_debug_info_for_correlation: - OS << "debug info for correlation is not necessary"; + case instrprof_error::unexpected_correlation_info: + OS << "debug info/binary for correlation is not necessary"; break; case instrprof_error::unable_to_correlate_profile: OS << "unable to correlate profile"; diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 8529a17987352..cf80a58f43bd9 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -27,14 +27,22 @@ using namespace llvm; /// Get profile section. Expected getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) { + // On COFF, the getInstrProfSectionName returns the section names may followed + // by "$M". The linker removes the dollar and everything after it in the final + // binary. Do the same to match. Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + auto StripSuffix = [ObjFormat](StringRef N) { + return ObjFormat == Triple::COFF ? N.split('$').first : N; + }; std::string ExpectedSectionName = getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); - for (auto &Section : Obj.sections()) + ExpectedSectionName = StripSuffix(ExpectedSectionName); + for (auto &Section : Obj.sections()) { if (auto SectionName = Section.getName()) - if (SectionName.get() == ExpectedSectionName) + if (*SectionName == ExpectedSectionName) return Section; + } return make_error( instrprof_error::unable_to_correlate_profile, "could not find section (" + Twine(ExpectedSectionName) + ")"); @@ -46,14 +54,38 @@ const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; llvm::Expected> InstrProfCorrelator::Context::get(std::unique_ptr Buffer, - const object::ObjectFile &Obj) { + const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind) { + auto C = std::make_unique(); auto CountersSection = getInstrProfSection(Obj, IPSK_cnts); if (auto Err = CountersSection.takeError()) return std::move(Err); - auto C = std::make_unique(); + if (FileKind == InstrProfCorrelator::BINARY) { + auto DataSection = getInstrProfSection(Obj, IPSK_covdata); + if (auto Err = DataSection.takeError()) + return std::move(Err); + auto DataOrErr = DataSection->getContents(); + if (!DataOrErr) + return DataOrErr.takeError(); + auto NameSection = getInstrProfSection(Obj, IPSK_covname); + if (auto Err = NameSection.takeError()) + return std::move(Err); + auto NameOrErr = NameSection->getContents(); + if (!NameOrErr) + return NameOrErr.takeError(); + C->DataStart = DataOrErr->data(); + C->DataEnd = DataOrErr->data() + DataOrErr->size(); + C->NameStart = NameOrErr->data(); + C->NameSize = NameOrErr->size(); + } C->Buffer = std::move(Buffer); C->CountersSectionStart = CountersSection->getAddress(); C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); + // In COFF object file, there's a null byte at the beginning of the counter + // section which doesn't exist in raw profile. + if (Obj.getTripleObjectFormat() == Triple::COFF) + ++C->CountersSectionStart; + C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; return Expected>(std::move(C)); } @@ -80,9 +112,17 @@ InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) { return get(std::move(*BufferOrErr), FileKind); } + if (FileKind == BINARY) { + auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); + if (auto Err = BufferOrErr.takeError()) + return std::move(Err); + + return get(std::move(*BufferOrErr), FileKind); + } return make_error( instrprof_error::unable_to_correlate_profile, - "unsupported correlation kind (only DWARF debug info is supported)"); + "unsupported correlation kind (only DWARF debug info and Binary format " + "(ELF/COFF) are supported)"); } llvm::Expected> @@ -93,7 +133,7 @@ InstrProfCorrelator::get(std::unique_ptr Buffer, return std::move(Err); if (auto *Obj = dyn_cast(BinOrErr->get())) { - auto CtxOrErr = Context::get(std::move(Buffer), *Obj); + auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind); if (auto Err = CtxOrErr.takeError()) return std::move(Err); auto T = Obj->makeTriple(); @@ -155,9 +195,11 @@ InstrProfCorrelatorImpl::get( instrprof_error::unable_to_correlate_profile, "unsupported debug info format (only DWARF is supported)"); } + if (Obj.isELF() || Obj.isCOFF()) + return std::make_unique>(std::move(Ctx)); return make_error( instrprof_error::unable_to_correlate_profile, - "unsupported correlation file type (only DWARF is supported)"); + "unsupported binary format (only ELF and COFF are supported)"); } template @@ -212,16 +254,16 @@ Error InstrProfCorrelatorImpl::dumpYaml(int MaxWarnings, } template -void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, - uint64_t CFGHash, - IntPtrT CounterOffset, - IntPtrT FunctionPtr, - uint32_t NumCounters) { +void InstrProfCorrelatorImpl::addDataProbe(uint64_t NameRef, + uint64_t CFGHash, + IntPtrT CounterOffset, + IntPtrT FunctionPtr, + uint32_t NumCounters) { // Check if a probe was already added for this counter offset. if (!CounterOffsets.insert(CounterOffset).second) return; Data.push_back({ - maybeSwap(IndexedInstrProf::ComputeHash(FunctionName)), + maybeSwap(NameRef), maybeSwap(CFGHash), // In this mode, CounterPtr actually stores the section relative address // of the counter. @@ -236,7 +278,6 @@ void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, // TODO: MC/DC is not yet supported. /*NumBitmapBytes=*/maybeSwap(0), }); - NamesVec.push_back(FunctionName.str()); } template @@ -349,6 +390,8 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( *FunctionName); LLVM_DEBUG(Die.dump(dbgs())); } + // In debug info correlation mode, the CounterPtr is an absolute address of + // the counter, but it's expected to be relative later when iterating Data. IntPtrT CounterOffset = *CounterPtr - CountersStart; if (Data) { InstrProfCorrelator::Probe P; @@ -366,8 +409,9 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( P.LineNumber = LineNumber; Data->Probes.push_back(P); } else { - this->addProbe(*FunctionName, *CFGHash, CounterOffset, - FunctionPtr.value_or(0), *NumCounters); + this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash, + CounterOffset, FunctionPtr.value_or(0), *NumCounters); + this->NamesVec.push_back(*FunctionName); } }; for (auto &CU : DICtx->normal_units()) @@ -394,3 +438,46 @@ Error DwarfInstrProfCorrelator::correlateProfileNameImpl() { /*doCompression=*/false, this->Names); return Result; } + +template +void BinaryInstrProfCorrelator::correlateProfileDataImpl( + int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) { + using RawProfData = RawInstrProf::ProfileData; + bool UnlimitedWarnings = (MaxWarnings == 0); + // -N suppressed warnings means we can emit up to N (unsuppressed) warnings + int NumSuppressedWarnings = -MaxWarnings; + + const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart; + const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd; + // We need to use < here because the last data record may have no padding. + for (const RawProfData *I = DataStart; I < DataEnd; ++I) { + uint64_t CounterPtr = this->template maybeSwap(I->CounterPtr); + uint64_t CountersStart = this->Ctx->CountersSectionStart; + uint64_t CountersEnd = this->Ctx->CountersSectionEnd; + if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) { + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + WithColor::warning() + << format("CounterPtr out of range for function: Actual=0x%x " + "Expected=[0x%x, 0x%x) at data offset=0x%x\n", + CounterPtr, CountersStart, CountersEnd, + (I - DataStart) * sizeof(RawProfData)); + } + } + // In binary correlation mode, the CounterPtr is an absolute address of the + // counter, but it's expected to be relative later when iterating Data. + IntPtrT CounterOffset = CounterPtr - CountersStart; + this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, + I->FunctionPointer, I->NumCounters); + } +} + +template +Error BinaryInstrProfCorrelator::correlateProfileNameImpl() { + if (this->Ctx->NameSize == 0) { + return make_error( + instrprof_error::unable_to_correlate_profile, + "could not find any profile data metadata in object file"); + } + this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize); + return Error::success(); +} diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index f3fd456ae7973..068922d421f8b 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -556,10 +556,6 @@ Error RawInstrProfReader::readHeader( "\nPLEASE update this tool to version in the raw profile, or " "regenerate raw profile with expected version.") .str()); - if (useCorrelate() && !Correlator) - return error(instrprof_error::missing_debug_info_for_correlation); - if (!useCorrelate() && Correlator) - return error(instrprof_error::unexpected_debug_info_for_correlation); uint64_t BinaryIdSize = swap(Header.BinaryIdsSize); // Binary id start just after the header if exists. @@ -607,8 +603,9 @@ Error RawInstrProfReader::readHeader( if (Correlator) { // These sizes in the raw file are zero because we constructed them in the // Correlator. - assert(DataSize == 0 && NamesSize == 0); - assert(CountersDelta == 0 && NamesDelta == 0); + if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 && + NamesDelta == 0)) + return error(instrprof_error::unexpected_correlation_info); Data = Correlator->getDataPointer(); DataEnd = Data + Correlator->getDataSize(); NamesStart = Correlator->getNamesPointer(); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 8674740001e58..fe5a0578bd972 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -64,10 +64,24 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { -cl::opt - DebugInfoCorrelate("debug-info-correlate", - cl::desc("Use debug info to correlate profiles."), - cl::init(false)); +// TODO: Remove -debug-info-correlate in next LLVM release, in favor of +// -profile-correlate=debug-info. +cl::opt DebugInfoCorrelate( + "debug-info-correlate", + cl::desc("Use debug info to correlate profiles. (Deprecated, use " + "-profile-correlate=debug-info)"), + cl::init(false)); + +cl::opt ProfileCorrelate( + "profile-correlate", + cl::desc("Use debug info or binary file to correlate profiles."), + cl::init(InstrProfCorrelator::NONE), + cl::values(clEnumValN(InstrProfCorrelator::NONE, "", + "No profile correlation"), + clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", + "Use debug info to correlate"), + clEnumValN(InstrProfCorrelator::BINARY, "binary", + "Use binary to correlate"))); } // namespace llvm namespace { @@ -792,7 +806,7 @@ void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { // in lightweight mode. We need to move the value profile pointer to the // Counter struct to get this working. assert( - !DebugInfoCorrelate && + !DebugInfoCorrelate && ProfileCorrelate == InstrProfCorrelator::NONE && "Value profiling is not yet supported with lightweight instrumentation"); GlobalVariable *Name = Ind->getName(); auto It = ProfileDataMap.find(Name); @@ -1219,8 +1233,9 @@ GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, // Use internal rather than private linkage so the counter variable shows up // in the symbol table when using debug info for correlation. - if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && - Linkage == GlobalValue::PrivateLinkage) + if ((DebugInfoCorrelate || + ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) && + TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage) Linkage = GlobalValue::InternalLinkage; // Due to the limitation of binder as of 2021/09/28, the duplicate weak @@ -1341,7 +1356,8 @@ InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); PD.RegionCounters = CounterPtr; - if (DebugInfoCorrelate) { + if (DebugInfoCorrelate || + ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { @@ -1386,7 +1402,7 @@ InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { // When debug information is correlated to profile data, a data variable // is not needed. - if (DebugInfoCorrelate) + if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) return; GlobalVariable *NamePtr = Inc->getName(); @@ -1484,20 +1500,28 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { } auto *Data = new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); - // Reference the counter variable with a label difference (link-time - // constant). - auto *RelativeCounterPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), - ConstantExpr::getPtrToInt(Data, IntPtrTy)); - - // Bitmaps are relative to the same data variable as profile counters. + Constant *RelativeCounterPtr; GlobalVariable *BitmapPtr = PD.RegionBitmaps; Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); - - if (BitmapPtr != nullptr) { - RelativeBitmapPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + InstrProfSectKind DataSectionKind; + // With binary profile correlation, profile data is not loaded into memory. + // profile data must reference profile counter with an absolute relocation. + if (ProfileCorrelate == InstrProfCorrelator::BINARY) { + DataSectionKind = IPSK_covdata; + RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy); + if (BitmapPtr != nullptr) + RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy); + } else { + // Reference the counter variable with a label difference (link-time + // constant). + DataSectionKind = IPSK_data; + RelativeCounterPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); + if (BitmapPtr != nullptr) + RelativeBitmapPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); } Constant *DataVals[] = { @@ -1507,7 +1531,8 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { Data->setInitializer(ConstantStruct::get(DataTy, DataVals)); Data->setVisibility(Visibility); - Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); + Data->setSection( + getInstrProfSectionName(DataSectionKind, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); maybeSetComdat(Data, Fn, CntsVarName); @@ -1595,7 +1620,9 @@ void InstrLowerer::emitNameData() { NamesSize = CompressedNameStr.size(); setGlobalVariableLargeSection(TT, *NamesVar); NamesVar->setSection( - getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); + ProfileCorrelate == InstrProfCorrelator::BINARY + ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat()) + : getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); // On COFF, it's important to reduce the alignment down to 1 to prevent the // linker from inserting padding before the start of the names section or // between names entries. diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 57ff1648788f9..3a57709c4e8b7 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -327,6 +327,7 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; +extern cl::opt ProfileCorrelate; } // namespace llvm static cl::opt @@ -381,7 +382,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) { ProfileVersion |= VARIANT_MASK_CSIR_PROF; if (PGOInstrumentEntry) ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; - if (DebugInfoCorrelate) + if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) ProfileVersion |= VARIANT_MASK_DBG_CORRELATE; if (PGOFunctionEntryCoverage) ProfileVersion |= diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll index 1401d8f620b3f..bbf895ea4b34e 100644 --- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll @@ -1,11 +1,19 @@ ; RUN: opt < %s -passes=instrprof -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s --check-prefix=BINARY target triple = "aarch64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) +; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), @__profn_bar = private constant [3 x i8] c"bar" ; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) +; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), + +; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names" +; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames" define void @_Z3foov() { call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 1, i32 0) diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 088138f27de58..322b7da2678f4 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -46,6 +46,7 @@ #include using namespace llvm; +using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind; // https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations // on each subcommand. @@ -124,6 +125,11 @@ cl::opt DebugInfoFilename( "the functions it found. For merge, use the provided debug info to " "correlate the raw profile."), cl::sub(ShowSubcommand), cl::sub(MergeSubcommand)); +cl::opt + BinaryFilename("binary-file", cl::init(""), + cl::desc("For merge, use the provided unstripped bianry to " + "correlate the raw profile."), + cl::sub(MergeSubcommand)); cl::opt FuncNameFilter( "function", cl::desc("Details for matching functions. For overlapping CSSPGO, this " @@ -787,14 +793,27 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, OutputFormat != PF_Text) exitWithError("unknown format is specified"); - std::unique_ptr Correlator; + // TODO: Maybe we should support correlation with mixture of different + // correlation modes(w/wo debug-info/object correlation). + if (!DebugInfoFilename.empty() && !BinaryFilename.empty()) + exitWithError("Expected only one of -debug-info, -binary-file"); + std::string CorrelateFilename; + ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE; if (!DebugInfoFilename.empty()) { - if (auto Err = InstrProfCorrelator::get(DebugInfoFilename, - InstrProfCorrelator::DEBUG_INFO) + CorrelateFilename = DebugInfoFilename; + CorrelateKind = ProfCorrelatorKind::DEBUG_INFO; + } else if (!BinaryFilename.empty()) { + CorrelateFilename = BinaryFilename; + CorrelateKind = ProfCorrelatorKind::BINARY; + } + + std::unique_ptr Correlator; + if (CorrelateKind != InstrProfCorrelator::NONE) { + if (auto Err = InstrProfCorrelator::get(CorrelateFilename, CorrelateKind) .moveInto(Correlator)) - exitWithError(std::move(Err), DebugInfoFilename); + exitWithError(std::move(Err), CorrelateFilename); if (auto Err = Correlator->correlateProfileData(MaxDbgCorrelationWarnings)) - exitWithError(std::move(Err), DebugInfoFilename); + exitWithError(std::move(Err), CorrelateFilename); } std::mutex ErrorLock;