From 3a394ce5d4d7d91251337bd0a2c1c1a074eb37e6 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Tue, 17 Oct 2023 19:24:12 -0400 Subject: [PATCH 01/10] [Profile] Add binary profile correlation. --- clang/lib/CodeGen/BackendUtil.cpp | 10 +- clang/lib/CodeGen/CoverageMappingGen.cpp | 20 ++ .../CodeGen/coverage-profile-raw-version.c | 9 + compiler-rt/include/profile/InstrProfData.inc | 2 + compiler-rt/lib/profile/InstrProfiling.h | 3 + .../lib/profile/InstrProfilingBuffer.c | 14 ++ compiler-rt/lib/profile/InstrProfilingMerge.c | 8 +- .../profile/InstrProfilingPlatformWindows.c | 2 - .../lib/profile/InstrProfilingWriter.c | 24 +-- compiler-rt/test/CMakeLists.txt | 5 +- .../Darwin/instrprof-debug-info-correlate.c | 4 +- .../instrprof-debug-info-correlate-warnings.c | 2 +- .../Linux/instrprof-debug-info-correlate.c | 6 +- .../instrprof-show-debug-info-correlation.c | 6 +- .../test/profile/instrprof-binary-correlate.c | 29 +++ llvm/include/llvm/ProfileData/InstrProf.h | 6 +- .../llvm/ProfileData/InstrProfCorrelator.h | 59 ++++- .../llvm/ProfileData/InstrProfData.inc | 2 + .../llvm/ProfileData/InstrProfReader.h | 10 + .../Instrumentation/PGOInstrumentation.h | 2 - .../CodeGen/TargetLoweringObjectFileImpl.cpp | 23 +- .../Coverage/CoverageMappingReader.cpp | 28 ++- llvm/lib/ProfileData/InstrProf.cpp | 8 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 203 ++++++++++++++---- llvm/lib/ProfileData/InstrProfReader.cpp | 8 +- .../Instrumentation/InstrProfiling.cpp | 31 +-- .../Instrumentation/PGOInstrumentation.cpp | 3 +- .../debug-info-correlate-coverage.ll | 2 +- .../InstrProfiling/debug-info-correlate.ll | 2 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 37 +++- 30 files changed, 436 insertions(+), 132 deletions(-) create mode 100644 clang/test/CodeGen/coverage-profile-raw-version.c create mode 100644 compiler-rt/test/profile/instrprof-binary-correlate.c diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 70accce456d3c0..dad3c9a145b504 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -42,6 +42,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" @@ -98,13 +99,16 @@ extern cl::opt PrintPipelinePasses; static cl::opt ClSanitizeOnOptimizerEarlyEP( "sanitizer-early-opt-ep", cl::Optional, cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); -} + +extern cl::opt ProfileCorrelate; +} // namespace llvm namespace { // Default filename used for profile generation. std::string getDefaultProfileGenName() { - return DebugInfoCorrelate ? "default_%m.proflite" : "default_%m.profraw"; + return ProfileCorrelate.getNumOccurrences() ? "default_%m.proflite" + : "default_%m.profraw"; } class EmitAssemblyHelper { @@ -197,7 +201,7 @@ class EmitAssemblyHelper { void EmitAssembly(BackendAction Action, std::unique_ptr OS); }; -} +} // namespace static SanitizerCoverageOptions getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) { diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index 76ed10091b025b..daff0d1d50923f 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -31,6 +31,10 @@ // is textually included. #define COVMAP_V3 +namespace llvm { +extern cl::opt ProfileCorrelate; +} // namespace llvm + static llvm::cl::opt EmptyLineCommentCoverage( "emptyline-comment-coverage", llvm::cl::desc("Emit emptylines and comment lines as skipped regions (only " @@ -1831,6 +1835,22 @@ void CoverageMappingModuleGen::emit() { llvm::GlobalValue::InternalLinkage, NamesArrVal, llvm::getCoverageUnusedNamesVarName()); } + const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); + llvm::Type *IntTy64 = llvm::Type::getInt64Ty(Ctx); + uint64_t ProfileVersion = INSTR_PROF_RAW_VERSION; + if (llvm::ProfileCorrelate == llvm::InstrProfCorrelator::BINARY) + ProfileVersion |= VARIANT_MASK_BIN_CORRELATE; + auto *VersionVariable = new llvm::GlobalVariable( + CGM.getModule(), llvm::Type::getInt64Ty(Ctx), true, + llvm::GlobalValue::WeakAnyLinkage, + llvm::Constant::getIntegerValue(IntTy64, llvm::APInt(64, ProfileVersion)), + VarName); + VersionVariable->setVisibility(llvm::GlobalValue::HiddenVisibility); + llvm::Triple TT(CGM.getModule().getTargetTriple()); + if (TT.supportsCOMDAT()) { + VersionVariable->setLinkage(llvm::GlobalValue::ExternalLinkage); + VersionVariable->setComdat(CGM.getModule().getOrInsertComdat(VarName)); + } } unsigned CoverageMappingModuleGen::getFileID(FileEntryRef File) { diff --git a/clang/test/CodeGen/coverage-profile-raw-version.c b/clang/test/CodeGen/coverage-profile-raw-version.c new file mode 100644 index 00000000000000..fc81179e1a33e0 --- /dev/null +++ b/clang/test/CodeGen/coverage-profile-raw-version.c @@ -0,0 +1,9 @@ +// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -mllvm -profile-correlate=binary -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s --check-prefix=BIN-CORRELATE + +// CHECK: @__llvm_profile_raw_version = {{.*}} i64 8 +// BIN-CORRELATE: @__llvm_profile_raw_version = {{.*}} i64 4294967304 + +int main() { + return 0; +} diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 8ba7e186d4fb1a..1841ef348a7006 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -655,6 +655,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the 8th most significant bit * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 32 indicates there is no profile name and data sections. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. @@ -664,6 +665,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_BIN_CORRELATE (0x1ULL << 32) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 4433d7bd48871f..8b9e029ee8baee 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -267,6 +267,9 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); +/*! \brief Get the size of the profile name section in bytes. */ +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); + /*! \brief Get the size in bytes of a single counter entry. */ size_t __llvm_profile_counter_entry_size(void); diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 61ac5d9c028500..e60aa9fc04e7fa 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -53,6 +53,9 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { + if ((__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) || + (__llvm_profile_get_version() & VARIANT_MASK_BIN_CORRELATE)) + return 0; intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); @@ -61,9 +64,20 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { + if ((__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) || + (__llvm_profile_get_version() & VARIANT_MASK_BIN_CORRELATE)) + return 0; return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) { + if ((__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) || + (__llvm_profile_get_version() & VARIANT_MASK_BIN_CORRELATE)) + return 0; + return End - Begin; +} + COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) return sizeof(uint8_t); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 9cf12f251f7262..554ce3693b6e46 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -66,8 +66,9 @@ int __llvm_profile_check_compatibility(const char *ProfileData, Header->NumCounters != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || - Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - - __llvm_profile_begin_names()) || + Header->NamesSize != + __llvm_profile_get_name_size(__llvm_profile_begin_names(), + __llvm_profile_end_names()) || Header->ValueKindLast != IPVK_Last) return 1; @@ -132,7 +133,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // Merge counters by iterating the entire counter section when debug info // correlation is enabled. - if (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) { + if ((__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) || + (__llvm_profile_get_version() & VARIANT_MASK_BIN_CORRELATE)) { for (SrcCounter = SrcCountersStart, DstCounter = __llvm_profile_begin_counters(); SrcCounter < SrcCountersEnd;) { diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index dd576b2f8357db..23a7739355e2ed 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,8 +13,6 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") -#pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 1e22398a4c0f64..f3709694427d5c 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -259,19 +259,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const char *CountersBegin, const char *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { - int DebugInfoCorrelate = - (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; + int ProfileCorrelation = + (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) || + (__llvm_profile_get_version() & VARIANT_MASK_BIN_CORRELATE); /* Calculate size of sections. */ const uint64_t DataSectionSize = - DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t NumData = - DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd); + __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); const uint64_t CountersSectionSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); - const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; + const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); /* Create the header. */ __llvm_profile_header Header; @@ -298,7 +298,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, #endif /* The data and names sections are omitted in lightweight mode. */ - if (DebugInfoCorrelate) { + if (ProfileCorrelation) { Header.CountersDelta = 0; Header.NamesDelta = 0; } @@ -314,19 +314,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Write the profile data. */ ProfDataIOVec IOVecData[] = { - {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize, + {ProfileCorrelation ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, - {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, + {(SkipNameDataWrite || ProfileCorrelation) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; - - /* Value profiling is not yet supported in continuous mode. */ - if (__llvm_profile_is_continuous_mode_enabled()) + /* Value profiling is not yet supported in continuous mode and profile + * correlation mode. */ + if (__llvm_profile_is_continuous_mode_enabled() || ProfileCorrelation) return 0; return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index f9b01b15b0e62c..7357604b1f651e 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -37,8 +37,9 @@ if(NOT ANDROID) if(NOT COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD) # Use LLVM utils and Clang from the same build tree. list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS - clang clang-resource-headers FileCheck count not llvm-config llvm-nm llvm-objdump - llvm-readelf llvm-readobj llvm-size llvm-symbolizer compiler-rt-headers sancov split-file) + clang clang-resource-headers FileCheck count not llvm-config llvm-nm + llvm-objdump llvm-readelf llvm-readobj llvm-size llvm-symbolizer + compiler-rt-headers sancov split-file llvm-strip) if (WIN32) list(APPEND SANITIZER_COMMON_LIT_TEST_DEPS KillTheDoctor) endif() diff --git a/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c index f347d439e2e067..46d25a4e386dc3 100644 --- a/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c +++ b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c @@ -1,5 +1,5 @@ // Value profiling is currently not supported in lightweight mode. -// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t.dSYM %t.proflite @@ -9,7 +9,7 @@ // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata) -// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov // RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov.dSYM %t.cov.proflite diff --git a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c index 5069c6340b64fd..25022f241a6d28 100644 --- a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c +++ b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c @@ -1,6 +1,6 @@ // Disable full debug info and verify that we get warnings during merging -// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=2 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,LIMIT --implicit-check-not=warning // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=0 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,NOLIMIT --implicit-check-not=warning diff --git a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c index a918d7b6299005..ccfd65b6f4c4b1 100644 --- a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c +++ b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c @@ -3,19 +3,19 @@ // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal // RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw -// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.d4.proflite %run %t.d4 // RUN: llvm-profdata merge -o %t.d4.profdata --debug-info=%t.d4 %t.d4.proflite // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.d4.profdata) -// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata) -// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov // RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov %t.cov.proflite diff --git a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c index 226d678aca347a..93bf40f98d3ab6 100644 --- a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c +++ b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c @@ -1,10 +1,10 @@ -// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s +// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s // RUN: llvm-profdata show --debug-info=%t --detailed-summary --show-prof-sym-list | FileCheck %s // RUN: llvm-profdata show --debug-info=%t --show-format=yaml | FileCheck %s --match-full-lines --check-prefix YAML -// RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s +// RUN: %clang_pgogen -o %t.no.dbg -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s // RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG -// NO-DBG: unable to correlate profile: could not find any profile metadata in debug info +// NO-DBG: unable to correlate profile: could not find any profile data metadata in correlated file // YAML: Probes: // YAML: - Function Name: a diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c new file mode 100644 index 00000000000000..2154c13027e286 --- /dev/null +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -0,0 +1,29 @@ +// REQUIRES: linux || windows +// Default +// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw + +// With -profile-correlate=binary flag +// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe +// RUN: llvm-profdata merge -o %t-1.profdata --object-file=%t-1.exe %t-1.profraw +// RUN: diff %t.normal.profdata %t-1.profdata + +// Strip above binary and run +// RUN: llvm-strip %t-1.exe -o %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-2.profdata --object-file=%t-1.exe %t-2.profraw +// RUN: diff %t.normal.profdata %t-2.profdata + +// Online merging. +// RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal +// RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw +// RUN: rm -rf %t.profdir && mkdir %t.profdir +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe +// RUN: llvm-profdata merge -o %t-4.profdata --object-file=%t-1.exe %t.profdir +// RUN: diff %t.normal.merged.profdata %t-4.profdata + +// TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 9239c1a691eca1..00a7bafb54e89e 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -328,8 +328,8 @@ enum class instrprof_error { too_large, truncated, malformed, - missing_debug_info_for_correlation, - unexpected_debug_info_for_correlation, + missing_correlation_info, + unexpected_correlation_info, unable_to_correlate_profile, unknown_function, invalid_prof, @@ -460,7 +460,7 @@ class InstrProfSymtab { /// only initialize the symtab with reference to the data and /// the section base address. The decompression will be delayed /// until before it is used. See also \c create(StringRef) method. - Error create(object::SectionRef &Section); + Error create(object::SectionRef &Section, bool MightHasNullByte = false); /// This interface is used by reader of CoverageMapping test /// format. diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index dd062951f277c1..661570eaee1130 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -5,8 +5,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This file defines InstrProfCorrelator used to generate PGO profiles from -// raw profile data and debug info. +// This file defines InstrProfCorrelator used to generate PGO/coverage profiles +// from raw profile data and debug info/binary file. //===----------------------------------------------------------------------===// #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H @@ -31,8 +31,12 @@ class ObjectFile; /// to their functions. class InstrProfCorrelator { public: + /// Indicate if we should use the debug info or profile metadata sections to + /// correlate. + enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY }; + static llvm::Expected> - get(StringRef DebugInfoFilename); + get(StringRef Filename, ProfCorrelatorKind FileKind); /// Construct a ProfileData vector used to correlate raw instrumentation data /// to their functions. @@ -68,11 +72,18 @@ class InstrProfCorrelator { protected: struct Context { static llvm::Expected> - get(std::unique_ptr Buffer, const object::ObjectFile &Obj); + get(std::unique_ptr Buffer, const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind); std::unique_ptr Buffer; /// The address range of the __llvm_prf_cnts section. uint64_t CountersSectionStart; uint64_t CountersSectionEnd; + /// The pointer points to start/end of profile data/name sections if + /// FileKind is Binary. + const char *DataStart; + const char *DataEnd; + const char *NameStart; + size_t NameSize; /// True if target and host have different endian orders. bool ShouldSwapBytes; }; @@ -104,7 +115,7 @@ class InstrProfCorrelator { private: static llvm::Expected> - get(std::unique_ptr Buffer); + get(std::unique_ptr Buffer, ProfCorrelatorKind FileKind); const InstrProfCorrelatorKind Kind; }; @@ -128,7 +139,7 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator { static llvm::Expected>> get(std::unique_ptr Ctx, - const object::ObjectFile &Obj); + const object::ObjectFile &Obj, ProfCorrelatorKind FileKind); protected: std::vector> Data; @@ -138,21 +149,23 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator { int MaxWarnings, InstrProfCorrelator::CorrelationData *Data = nullptr) = 0; + virtual Error correlateProfileNameImpl() = 0; + Error dumpYaml(int MaxWarnings, raw_ostream &OS) override; void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, IntPtrT FunctionPtr, uint32_t NumCounters); + // Byte-swap the value if necessary. + template T maybeSwap(T Value) const { + return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; + } + private: InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind, std::unique_ptr Ctx) : InstrProfCorrelator(Kind, std::move(Ctx)){}; llvm::DenseSet CounterOffsets; - - // Byte-swap the value if necessary. - template T maybeSwap(T Value) const { - return Ctx->ShouldSwapBytes ? llvm::byteswap(Value) : Value; - } }; /// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes @@ -205,6 +218,30 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl { void correlateProfileDataImpl( int MaxWarnings, InstrProfCorrelator::CorrelationData *Data = nullptr) override; + + Error correlateProfileNameImpl() override; +}; + +/// BinaryInstrProfCorrelator - A child of InstrProfCorrelatorImpl that +/// takes an object file as input to correlate profiles. +template +class BinaryInstrProfCorrelator : public InstrProfCorrelatorImpl { +public: + BinaryInstrProfCorrelator(std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(std::move(Ctx)) {} + + /// Return a pointer to the names string that this class constructs. + const char *getNamesPointer() const { return this->Ctx.NameStart; } + + /// Return the number of bytes in the names string. + size_t getNamesSize() const { return this->Ctx.NameSize; } + +private: + void correlateProfileDataImpl( + int MaxWarnings, + InstrProfCorrelator::CorrelationData *Data = nullptr) override; + + Error correlateProfileNameImpl() override; }; } // end namespace llvm diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 13be2753e514ef..f6c5f2b3fa1730 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -655,6 +655,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the 8th most significant bit * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 32 indicates whether to use binary to correlate profiles. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. @@ -664,6 +665,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_BIN_CORRELATE (0x1ULL << 32) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 5f54cbeb1b01ed..d5eb009087d9f9 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -126,6 +126,8 @@ class InstrProfReader { /// Return true if we must provide debug info to create PGO profiles. virtual bool useDebugInfoCorrelate() const { return false; } + virtual bool useBinaryCorrelate() const { return false; } + /// Return true if the profile has single byte counters representing coverage. virtual bool hasSingleByteCoverage() const = 0; @@ -374,6 +376,14 @@ class RawInstrProfReader : public InstrProfReader { return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; } + bool useBinaryCorrelate() const override { + return (Version & VARIANT_MASK_BIN_CORRELATE) != 0; + } + + bool useCorrelate() const { + return useDebugInfoCorrelate() || useBinaryCorrelate(); + } + bool hasSingleByteCoverage() const override { return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; } diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h index 5b1977b7de9a2a..3d8f3bd9235451 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -24,8 +24,6 @@ namespace llvm { -extern cl::opt DebugInfoCorrelate; - class Function; class Instruction; class Module; diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 6210e7fc128a30..fd785ff2629932 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -57,6 +57,7 @@ #include "llvm/MC/MCValue.h" #include "llvm/MC/SectionKind.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/Base64.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" @@ -71,6 +72,10 @@ using namespace llvm; using namespace dwarf; +namespace llvm { +extern cl::opt ProfileCorrelate; +} // namespace llvm + static cl::opt JumpTableInFunctionSection( "jumptable-in-function-section", cl::Hidden, cl::init(false), cl::desc("Putting Jump Table in function section")); @@ -472,7 +477,15 @@ static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) { /*AddSegmentInfo=*/false) || Name == getInstrProfSectionName(IPSK_covfun, Triple::ELF, /*AddSegmentInfo=*/false) || - Name == ".llvmbc" || Name == ".llvmcmd") + Name == ".llvmbc" || Name == ".llvmcmd" || + // Instead of checking the flag, maybe we should check if + // VARIANT_MASK_BIN_CORRELATE flag is set at the profile version global + // variable. + (ProfileCorrelate == InstrProfCorrelator::BINARY && + (Name == getInstrProfSectionName(llvm::IPSK_name, Triple::ELF, + /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(llvm::IPSK_data, Triple::ELF, + /*AddSegmentInfo=*/false)))) return SectionKind::getMetadata(); if (Name.empty() || Name[0] != '.') return K; @@ -1674,9 +1687,15 @@ static int getSelectionForCOFF(const GlobalValue *GV) { MCSection *TargetLoweringObjectFileCOFF::getExplicitSectionGlobal( const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const { + StringRef Name = GO->getSection(); + if (ProfileCorrelate == InstrProfCorrelator::BINARY && + (Name == getInstrProfSectionName(llvm::IPSK_name, Triple::COFF, + /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(llvm::IPSK_data, Triple::COFF, + /*AddSegmentInfo=*/false))) + Kind = SectionKind::getMetadata(); int Selection = 0; unsigned Characteristics = getCOFFSectionFlags(Kind, TM); - StringRef Name = GO->getSection(); StringRef COMDATSymName = ""; if (GO->hasComdat()) { Selection = getSelectionForCOFF(GO); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 857498256ec54c..a56e4c90911141 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -457,7 +457,7 @@ Expected RawCoverageMappingDummyChecker::isDummy() { return Tag == Counter::Zero; } -Error InstrProfSymtab::create(SectionRef &Section) { +Error InstrProfSymtab::create(SectionRef &Section, bool MightHasNullByte) { Expected DataOrErr = Section.getContents(); if (!DataOrErr) return DataOrErr.takeError(); @@ -467,7 +467,8 @@ Error InstrProfSymtab::create(SectionRef &Section) { // If this is a linked PE/COFF file, then we have to skip over the null byte // that is allocated in the .lprfn$A section in the LLVM profiling runtime. const ObjectFile *Obj = Section.getObject(); - if (isa(Obj) && !Obj->isRelocatableObject()) + if (MightHasNullByte && isa(Obj) && + !Obj->isRelocatableObject()) Data = Data.drop_front(1); return Error::success(); @@ -1070,11 +1071,24 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, InstrProfSymtab ProfileNames; std::vector NamesSectionRefs = *NamesSection; - if (NamesSectionRefs.size() != 1) - return make_error( - coveragemap_error::malformed, - "the size of coverage mapping section is not one"); - if (Error E = ProfileNames.create(NamesSectionRefs.back())) + bool MightHasNullByte = true; + if (NamesSectionRefs.size() != 1) { + // By default, the profile name section in the binary starts with a null + // byte and followed by names. But if binary correlation is enabled, there + // will be 2 name sections. One contains only two null bytes, and another + // one contains the names without the dummy null byte at the beginning. + if (ObjFormat == Triple::COFF) { + NamesSectionRefs.erase( + std::remove_if(NamesSectionRefs.begin(), NamesSectionRefs.end(), + [](const SectionRef &S) { return S.isData(); })); + MightHasNullByte = false; + } + if (NamesSectionRefs.size() != 1) + return make_error( + coveragemap_error::malformed, + "the size of coverage mapping section is not one"); + } + if (Error E = ProfileNames.create(NamesSectionRefs.back(), MightHasNullByte)) return std::move(E); // Look for the coverage records section (Version4 only). diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index ddc11304742df7..349ebf3762df40 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -113,11 +113,11 @@ static std::string getInstrProfErrString(instrprof_error Err, case instrprof_error::malformed: OS << "malformed instrumentation profile data"; break; - case instrprof_error::missing_debug_info_for_correlation: - OS << "debug info for correlation is required"; + case instrprof_error::missing_correlation_info: + OS << "debug info/binary for correlation is required"; break; - case instrprof_error::unexpected_debug_info_for_correlation: - OS << "debug info for correlation is not necessary"; + case instrprof_error::unexpected_correlation_info: + OS << "debug info/binary for correlation is not necessary"; break; case instrprof_error::unable_to_correlate_profile: OS << "unable to correlate profile"; diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 71787c9bd8577b..347879e71bbda1 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -24,15 +24,43 @@ using namespace llvm; -/// Get the __llvm_prf_cnts section. -Expected getCountersSection(const object::ObjectFile &Obj) { +namespace llvm { +cl::opt ProfileCorrelate( + "profile-correlate", + cl::desc("Use debug info or binary file to correlate profiles."), + cl::init(InstrProfCorrelator::NONE), + cl::values(clEnumValN(InstrProfCorrelator::NONE, "", + "No profile correlation"), + clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", + "Use debug info to correlate"), + clEnumValN(InstrProfCorrelator::BINARY, "binary", + "Use binary to correlate"))); +} // namespace llvm + +/// Get profile section. +Expected getInstrProfSection(const object::ObjectFile &Obj, + InstrProfSectKind IPSK) { + // On COFF, the object file section name may end in "$M". This tells the + // linker to sort these sections between "$A" and "$Z". The linker removes the + // dollar and everything after it in the final binary. Do the same to match. + Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + std::string ExpectedSectionName = + getInstrProfSectionName(IPSK, ObjFormat, + /*AddSegmentInfo=*/false); for (auto &Section : Obj.sections()) if (auto SectionName = Section.getName()) - if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) + if (SectionName.get() == ExpectedSectionName) { + // There will be extra profile name and data sections in COFF used for + // runtime to detect start/stop of those sections, skipping them. + // (details at compiler-rt/lib/profile/InstrProfilingPlatformWindows.c) + if (ObjFormat == Triple::COFF && IPSK != IPSK_cnts && Section.isData()) + continue; return Section; + } + return make_error( instrprof_error::unable_to_correlate_profile, - "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); + "could not find section (" + Twine(ExpectedSectionName) + ")"); } const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; @@ -41,56 +69,84 @@ const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; llvm::Expected> InstrProfCorrelator::Context::get(std::unique_ptr Buffer, - const object::ObjectFile &Obj) { - auto CountersSection = getCountersSection(Obj); + const object::ObjectFile &Obj, + ProfCorrelatorKind FileKind) { + auto C = std::make_unique(); + auto CountersSection = getInstrProfSection(Obj, IPSK_cnts); if (auto Err = CountersSection.takeError()) return std::move(Err); - auto C = std::make_unique(); + if (FileKind == InstrProfCorrelator::BINARY) { + auto DataSection = getInstrProfSection(Obj, IPSK_data); + if (auto Err = DataSection.takeError()) + return std::move(Err); + auto DataOrErr = DataSection->getContents(); + if (!DataOrErr) + return DataOrErr.takeError(); + auto NameSection = getInstrProfSection(Obj, IPSK_name); + if (auto Err = NameSection.takeError()) + return std::move(Err); + auto NameOrErr = NameSection->getContents(); + if (!NameOrErr) + return NameOrErr.takeError(); + C->DataStart = DataOrErr->data(); + C->DataEnd = DataOrErr->data() + DataOrErr->size(); + C->NameStart = NameOrErr->data(); + C->NameSize = NameOrErr->size(); + } C->Buffer = std::move(Buffer); C->CountersSectionStart = CountersSection->getAddress(); C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); + // In COFF object file, there's a null byte at the beginning of the counter + // section which doesn't exist in raw profile. + if (Obj.getTripleObjectFormat() == Triple::COFF) + C->CountersSectionStart++; + C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; return Expected>(std::move(C)); } llvm::Expected> -InstrProfCorrelator::get(StringRef DebugInfoFilename) { - auto DsymObjectsOrErr = - object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); - if (auto Err = DsymObjectsOrErr.takeError()) - return std::move(Err); - if (!DsymObjectsOrErr->empty()) { - // TODO: Enable profile correlation when there are multiple objects in a - // dSYM bundle. - if (DsymObjectsOrErr->size() > 1) - return make_error( - instrprof_error::unable_to_correlate_profile, - "using multiple objects is not yet supported"); - DebugInfoFilename = *DsymObjectsOrErr->begin(); +InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) { + if (FileKind == DEBUG_INFO) { + auto DsymObjectsOrErr = + object::MachOObjectFile::findDsymObjectMembers(Filename); + if (auto Err = DsymObjectsOrErr.takeError()) + return std::move(Err); + if (!DsymObjectsOrErr->empty()) { + // TODO: Enable profile correlation when there are multiple objects in a + // dSYM bundle. + if (DsymObjectsOrErr->size() > 1) + return make_error( + instrprof_error::unable_to_correlate_profile, + "using multiple objects is not yet supported"); + Filename = *DsymObjectsOrErr->begin(); + } } - auto BufferOrErr = - errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); + auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename)); if (auto Err = BufferOrErr.takeError()) return std::move(Err); - return get(std::move(*BufferOrErr)); + return get(std::move(*BufferOrErr), FileKind); } llvm::Expected> -InstrProfCorrelator::get(std::unique_ptr Buffer) { +InstrProfCorrelator::get(std::unique_ptr Buffer, + ProfCorrelatorKind FileKind) { auto BinOrErr = object::createBinary(*Buffer); if (auto Err = BinOrErr.takeError()) return std::move(Err); if (auto *Obj = dyn_cast(BinOrErr->get())) { - auto CtxOrErr = Context::get(std::move(Buffer), *Obj); + auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind); if (auto Err = CtxOrErr.takeError()) return std::move(Err); auto T = Obj->makeTriple(); if (T.isArch64Bit()) - return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj, + FileKind); if (T.isArch32Bit()) - return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj, + FileKind); } return make_error( instrprof_error::unable_to_correlate_profile, "not an object file"); @@ -132,29 +188,35 @@ template llvm::Expected>> InstrProfCorrelatorImpl::get( std::unique_ptr Ctx, - const object::ObjectFile &Obj) { - if (Obj.isELF() || Obj.isMachO()) { - auto DICtx = DWARFContext::create(Obj); - return std::make_unique>(std::move(DICtx), - std::move(Ctx)); + const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) { + if (FileKind == DEBUG_INFO) { + if (Obj.isELF() || Obj.isMachO()) { + auto DICtx = DWARFContext::create(Obj); + return std::make_unique>( + std::move(DICtx), std::move(Ctx)); + } + return make_error( + instrprof_error::unable_to_correlate_profile, + "unsupported debug info format (only DWARF is supported)"); } + if (Obj.isELF() || Obj.isCOFF()) + return std::make_unique>(std::move(Ctx)); return make_error( - instrprof_error::unable_to_correlate_profile, - "unsupported debug info format (only DWARF is supported)"); + instrprof_error::unable_to_correlate_profile, + "unsupported binary format (only ELF and COFF are supported)"); } template Error InstrProfCorrelatorImpl::correlateProfileData(int MaxWarnings) { assert(Data.empty() && Names.empty() && NamesVec.empty()); correlateProfileDataImpl(MaxWarnings); - if (Data.empty() || NamesVec.empty()) + if (this->Data.empty()) return make_error( instrprof_error::unable_to_correlate_profile, - "could not find any profile metadata in debug info"); - auto Result = - collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); - CounterOffsets.clear(); - NamesVec.clear(); + "could not find any profile data metadata in correlated file"); + Error Result = correlateProfileNameImpl(); + this->CounterOffsets.clear(); + this->NamesVec.clear(); return Result; } @@ -361,3 +423,64 @@ void DwarfInstrProfCorrelator::correlateProfileDataImpl( WithColor::warning() << format("Suppressed %d additional warnings\n", NumSuppressedWarnings); } + +template +Error DwarfInstrProfCorrelator::correlateProfileNameImpl() { + if (this->NamesVec.empty()) { + return make_error( + instrprof_error::unable_to_correlate_profile, + "could not find any profile name metadata in debug info"); + } + auto Result = collectPGOFuncNameStrings(this->NamesVec, + /*doCompression=*/false, this->Names); + return Result; +} + +template +void BinaryInstrProfCorrelator::correlateProfileDataImpl( + int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) { + bool UnlimitedWarnings = (MaxWarnings == 0); + // -N suppressed warnings means we can emit up to N (unsuppressed) warnings + int NumSuppressedWarnings = -MaxWarnings; + + const RawInstrProf::ProfileData *DataStart = + (const RawInstrProf::ProfileData *)this->Ctx->DataStart; + const RawInstrProf::ProfileData *DataEnd = + (const RawInstrProf::ProfileData *)this->Ctx->DataEnd; + // We need to use < here because the last data record may have no padding. + for (const RawInstrProf::ProfileData *I = DataStart; I < DataEnd; + ++I) { + uint64_t CounterPtr = this->template maybeSwap(I->CounterPtr); + uint64_t CountersStart = this->Ctx->CountersSectionStart; + uint64_t CountersEnd = this->Ctx->CountersSectionEnd; + if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) { + if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { + WithColor::warning() << format( + "CounterPtr out of range for function: Actual=0x%x " + "Expected=[0x%x, 0x%x) at data offset=0x%x\n", + CounterPtr, CountersStart, CountersEnd, + (I - DataStart) * sizeof(RawInstrProf::ProfileData)); + } + } + IntPtrT CounterOffset = CounterPtr - CountersStart; + this->Data.push_back({I->NameRef, + I->FuncHash, + this->template maybeSwap(CounterOffset), + I->FunctionPointer, + // Value profiling is not supported. + 0, + I->NumCounters, + {0, 0}}); + } +} + +template +Error BinaryInstrProfCorrelator::correlateProfileNameImpl() { + if (this->Ctx->NameSize == 0) { + return make_error( + instrprof_error::unable_to_correlate_profile, + "could not find any profile data metadata in object file"); + } + this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize); + return Error::success(); +} diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index a920a31d0a4b22..2b5181e6b34043 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -540,10 +540,10 @@ Error RawInstrProfReader::readHeader( "\nPLEASE update this tool to version in the raw profile, or " "regenerate raw profile with expected version.") .str()); - if (useDebugInfoCorrelate() && !Correlator) - return error(instrprof_error::missing_debug_info_for_correlation); - if (!useDebugInfoCorrelate() && Correlator) - return error(instrprof_error::unexpected_debug_info_for_correlation); + if (useCorrelate() && !Correlator) + return error(instrprof_error::missing_correlation_info); + if (!useCorrelate() && Correlator) + return error(instrprof_error::unexpected_correlation_info); BinaryIdsSize = swap(Header.BinaryIdsSize); if (BinaryIdsSize % sizeof(uint64_t)) diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 57fcfd53836911..7eaa92bfe842e1 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -60,10 +60,7 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { -cl::opt - DebugInfoCorrelate("debug-info-correlate", - cl::desc("Use debug info to correlate profiles."), - cl::init(false)); +extern cl::opt ProfileCorrelate; } // namespace llvm namespace { @@ -627,7 +624,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { // in lightweight mode. We need to move the value profile pointer to the // Counter struct to get this working. assert( - !DebugInfoCorrelate && + ProfileCorrelate == InstrProfCorrelator::NONE && "Value profiling is not yet supported with lightweight instrumentation"); GlobalVariable *Name = Ind->getName(); auto It = ProfileDataMap.find(Name); @@ -965,8 +962,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { // Use internal rather than private linkage so the counter variable shows up // in the symbol table when using debug info for correlation. - if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && - Linkage == GlobalValue::PrivateLinkage) + if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO && + TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage) Linkage = GlobalValue::InternalLinkage; // Due to the limitation of binder as of 2021/09/28, the duplicate weak @@ -1030,7 +1027,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { CounterPtr->setLinkage(Linkage); MaybeSetComdat(CounterPtr); PD.RegionCounters = CounterPtr; - if (DebugInfoCorrelate) { + if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { @@ -1083,7 +1080,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } - if (DebugInfoCorrelate) { + if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) { // Mark the counter variable as used so that it isn't optimized out. CompilerUsedVars.push_back(PD.RegionCounters); return PD.RegionCounters; @@ -1124,11 +1121,17 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { } auto *Data = new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); - // Reference the counter variable with a label difference (link-time - // constant). - auto *RelativeCounterPtr = - ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), - ConstantExpr::getPtrToInt(Data, IntPtrTy)); + Constant *RelativeCounterPtr; + if (ProfileCorrelate == InstrProfCorrelator::BINARY) { + // CounterPtr needs to be absolute when we make data section strippable. + RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy); + } else { + // Reference the counter variable with a label difference (link-time + // constant). + RelativeCounterPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); + } Constant *DataVals[] = { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 7ad1c9bc54f378..6dcbf3ada5f0c7 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -327,6 +327,7 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; +extern cl::opt ProfileCorrelate; } // namespace llvm static cl::opt @@ -381,7 +382,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) { ProfileVersion |= VARIANT_MASK_CSIR_PROF; if (PGOInstrumentEntry) ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; - if (DebugInfoCorrelate) + if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) ProfileVersion |= VARIANT_MASK_DBG_CORRELATE; if (PGOFunctionEntryCoverage) ProfileVersion |= diff --git a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll index 192bac6e503a07..dd646153381705 100644 --- a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=instrprof -debug-info-correlate -S | opt -O2 -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -profile-correlate=debug-info -S | opt -O2 -S | FileCheck %s @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo diff --git a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll index fd868ead5b78d2..84eaab33701a48 100644 --- a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll +++ b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=instrprof -debug-info-correlate -S > %t.ll +; RUN: opt < %s -passes=instrprof -profile-correlate=debug-info -S > %t.ll ; RUN: FileCheck < %t.ll --implicit-check-not "{{__llvm_prf_data|__llvm_prf_names}}" %s ; RUN: %llc_dwarf -O0 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s --check-prefix CHECK-DWARF diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 7d665a8005b0d6..c0db32e8343d82 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -413,10 +413,11 @@ static void writeInstrProfile(StringRef OutputFilename, static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, - SymbolRemapper *Remapper, StringRef OutputFilename, - ProfileFormat OutputFormat, uint64_t TraceReservoirSize, - uint64_t MaxTraceLength, int MaxDbgCorrelationWarnings, - bool OutputSparse, unsigned NumThreads, FailureMode FailMode, + StringRef ObjectFilename, SymbolRemapper *Remapper, + StringRef OutputFilename, ProfileFormat OutputFormat, + uint64_t TraceReservoirSize, uint64_t MaxTraceLength, + int MaxDbgCorrelationWarnings, bool OutputSparse, + unsigned NumThreads, FailureMode FailMode, const StringRef ProfiledBinary) { if (OutputFormat == PF_Compact_Binary) exitWithError("Compact Binary is deprecated"); @@ -425,12 +426,17 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, exitWithError("unknown format is specified"); std::unique_ptr Correlator; - if (!DebugInfoFilename.empty()) { + if (!DebugInfoFilename.empty() || !ObjectFilename.empty()) { + InstrProfCorrelator::ProfCorrelatorKind Kind = + DebugInfoFilename.empty() ? InstrProfCorrelator::BINARY + : InstrProfCorrelator::DEBUG_INFO; + StringRef FileName = + DebugInfoFilename.empty() ? ObjectFilename : DebugInfoFilename; if (auto Err = - InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator)) - exitWithError(std::move(Err), DebugInfoFilename); + InstrProfCorrelator::get(FileName, Kind).moveInto(Correlator)) + exitWithError(std::move(Err), FileName); if (auto Err = Correlator->correlateProfileData(MaxDbgCorrelationWarnings)) - exitWithError(std::move(Err), DebugInfoFilename); + exitWithError(std::move(Err), FileName); } std::mutex ErrorLock; @@ -1279,6 +1285,10 @@ static int merge_main(int argc, const char *argv[]) { cl::opt DebugInfoFilename( "debug-info", cl::init(""), cl::desc("Use the provided debug info to correlate the raw profile.")); + cl::opt ObjectFilename( + "object-file", cl::init(""), + cl::desc("Read and extract profile metadata from object file and show " + "the functions it found.")); cl::opt MaxDbgCorrelationWarnings( "max-debug-info-correlation-warnings", cl::desc("The maximum number of warnings to emit when correlating " @@ -1304,6 +1314,9 @@ static int merge_main(int argc, const char *argv[]) { "(default: 10000)")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); + if (!DebugInfoFilename.empty() && !ObjectFilename.empty()) { + exitWithError("Expected only one of -debug-info, -object-file"); + } WeightedFileVector WeightedInputs; for (StringRef Filename : InputFilenames) @@ -1342,8 +1355,8 @@ static int merge_main(int argc, const char *argv[]) { } if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), - OutputFilename, OutputFormat, + mergeInstrProfile(WeightedInputs, DebugInfoFilename, ObjectFilename, + Remapper.get(), OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, TemporalProfMaxTraceLength, MaxDbgCorrelationWarnings, OutputSparse, NumThreads, FailureMode, ProfiledBinary); @@ -2893,7 +2906,9 @@ static int showDebugInfoCorrelation(const std::string &Filename, if (SFormat == ShowFormat::Json) exitWithError("JSON output is not supported for debug info correlation"); std::unique_ptr Correlator; - if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator)) + if (auto Err = + InstrProfCorrelator::get(Filename, InstrProfCorrelator::DEBUG_INFO) + .moveInto(Correlator)) exitWithError(std::move(Err), Filename); if (SFormat == ShowFormat::Yaml) { if (auto Err = Correlator->dumpYaml(MaxDbgCorrelationWarnings, OS)) From 776784459f67f44c0455ab0f3b216f7d9c6a6376 Mon Sep 17 00:00:00 2001 From: My Name Date: Wed, 18 Oct 2023 13:30:21 -0400 Subject: [PATCH 02/10] fix correlator for coff. --- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 347879e71bbda1..168c2803145f1f 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -44,12 +44,16 @@ Expected getInstrProfSection(const object::ObjectFile &Obj, // linker to sort these sections between "$A" and "$Z". The linker removes the // dollar and everything after it in the final binary. Do the same to match. Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + auto StripSuffix = [ObjFormat](StringRef N) { + return ObjFormat == Triple::COFF ? N.split('$').first : N; + }; std::string ExpectedSectionName = getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); + ExpectedSectionName = StripSuffix(ExpectedSectionName); for (auto &Section : Obj.sections()) if (auto SectionName = Section.getName()) - if (SectionName.get() == ExpectedSectionName) { + if (StripSuffix(*SectionName) == ExpectedSectionName) { // There will be extra profile name and data sections in COFF used for // runtime to detect start/stop of those sections, skipping them. // (details at compiler-rt/lib/profile/InstrProfilingPlatformWindows.c) From c5cca07062572077c4140b4eb9a4bdbb22a385a6 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 18 Oct 2023 13:59:04 -0400 Subject: [PATCH 03/10] update docs and renaming option --- .../test/profile/instrprof-binary-correlate.c | 6 +++--- llvm/docs/CommandGuide/llvm-profdata.rst | 13 +++++++++---- llvm/tools/llvm-profdata/llvm-profdata.cpp | 18 +++++++++--------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c index 2154c13027e286..bc05dbd99b8251 100644 --- a/compiler-rt/test/profile/instrprof-binary-correlate.c +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -7,13 +7,13 @@ // With -profile-correlate=binary flag // RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe -// RUN: llvm-profdata merge -o %t-1.profdata --object-file=%t-1.exe %t-1.profraw +// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw // RUN: diff %t.normal.profdata %t-1.profdata // Strip above binary and run // RUN: llvm-strip %t-1.exe -o %t-2.exe // RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe -// RUN: llvm-profdata merge -o %t-2.profdata --object-file=%t-1.exe %t-2.profraw +// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw // RUN: diff %t.normal.profdata %t-2.profdata // Online merging. @@ -23,7 +23,7 @@ // RUN: rm -rf %t.profdir && mkdir %t.profdir // RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe // RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe -// RUN: llvm-profdata merge -o %t-4.profdata --object-file=%t-1.exe %t.profdir +// RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir // RUN: diff %t.normal.merged.profdata %t-4.profdata // TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst index be42733ca14056..f81c80601e98a1 100644 --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -195,8 +195,13 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to correlate the raw profile. + When ``-profile-correlate=debug-info`` was used for instrumentation, use this + option to correlate the raw profile. + +.. option:: --binary-file= +Specify the executable that contains profile data and profile name sections for +the raw profile. When ``-profile-correlate=binary`` was used for +instrumentation, use this option to correlate the raw profile. .. option:: --temporal-profile-trace-reservoir-size @@ -346,8 +351,8 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-debug-info-correlate`` was used for instrumentation, use this option - to show the correlated functions from the raw profile. + When ``-profile-correlate=debug-info`` was used for instrumentation, use this + option to show the correlated functions from the raw profile. .. option:: --covered diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index c0db32e8343d82..068504228d97d3 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -413,7 +413,7 @@ static void writeInstrProfile(StringRef OutputFilename, static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, - StringRef ObjectFilename, SymbolRemapper *Remapper, + StringRef BinaryFilename, SymbolRemapper *Remapper, StringRef OutputFilename, ProfileFormat OutputFormat, uint64_t TraceReservoirSize, uint64_t MaxTraceLength, int MaxDbgCorrelationWarnings, bool OutputSparse, @@ -426,12 +426,12 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, exitWithError("unknown format is specified"); std::unique_ptr Correlator; - if (!DebugInfoFilename.empty() || !ObjectFilename.empty()) { + if (!DebugInfoFilename.empty() || !BinaryFilename.empty()) { InstrProfCorrelator::ProfCorrelatorKind Kind = DebugInfoFilename.empty() ? InstrProfCorrelator::BINARY : InstrProfCorrelator::DEBUG_INFO; StringRef FileName = - DebugInfoFilename.empty() ? ObjectFilename : DebugInfoFilename; + DebugInfoFilename.empty() ? BinaryFilename : DebugInfoFilename; if (auto Err = InstrProfCorrelator::get(FileName, Kind).moveInto(Correlator)) exitWithError(std::move(Err), FileName); @@ -1285,9 +1285,9 @@ static int merge_main(int argc, const char *argv[]) { cl::opt DebugInfoFilename( "debug-info", cl::init(""), cl::desc("Use the provided debug info to correlate the raw profile.")); - cl::opt ObjectFilename( - "object-file", cl::init(""), - cl::desc("Read and extract profile metadata from object file and show " + cl::opt BinaryFilename( + "binary-file", cl::init(""), + cl::desc("Read and extract profile metadata from binary file and show " "the functions it found.")); cl::opt MaxDbgCorrelationWarnings( "max-debug-info-correlation-warnings", @@ -1314,8 +1314,8 @@ static int merge_main(int argc, const char *argv[]) { "(default: 10000)")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); - if (!DebugInfoFilename.empty() && !ObjectFilename.empty()) { - exitWithError("Expected only one of -debug-info, -object-file"); + if (!DebugInfoFilename.empty() && !BinaryFilename.empty()) { + exitWithError("Expected only one of -debug-info, -binary-file"); } WeightedFileVector WeightedInputs; @@ -1355,7 +1355,7 @@ static int merge_main(int argc, const char *argv[]) { } if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, DebugInfoFilename, ObjectFilename, + mergeInstrProfile(WeightedInputs, DebugInfoFilename, BinaryFilename, Remapper.get(), OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, TemporalProfMaxTraceLength, MaxDbgCorrelationWarnings, From 44704f6fd6ec0f277ed734b2d91c15b1fb3a7fe6 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 18 Oct 2023 14:48:54 -0400 Subject: [PATCH 04/10] format --- clang/lib/CodeGen/BackendUtil.cpp | 2 +- compiler-rt/test/profile/instrprof-binary-correlate.c | 8 ++++---- llvm/docs/CommandGuide/llvm-profdata.rst | 7 ++++--- llvm/include/llvm/ProfileData/InstrProfCorrelator.h | 2 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index dad3c9a145b504..69692c28b16cba 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -56,6 +56,7 @@ #include "llvm/Target/TargetOptions.h" #include "llvm/TargetParser/SubtargetFeature.h" #include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/HipStdPar/HipStdPar.h" #include "llvm/Transforms/IPO/EmbedBitcodePass.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" @@ -79,7 +80,6 @@ #include "llvm/Transforms/Scalar/EarlyCSE.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Scalar/JumpThreading.h" -#include "llvm/Transforms/HipStdPar/HipStdPar.h" #include "llvm/Transforms/Utils/Debugify.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/ModuleUtils.h" diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c index bc05dbd99b8251..d921c4c5dcb019 100644 --- a/compiler-rt/test/profile/instrprof-binary-correlate.c +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -7,14 +7,14 @@ // With -profile-correlate=binary flag // RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe -// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw -// RUN: diff %t.normal.profdata %t-1.profdata +// RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw +// RUN: diff %t.normal.profdata %t-1.profdata // Strip above binary and run // RUN: llvm-strip %t-1.exe -o %t-2.exe // RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe -// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw -// RUN: diff %t.normal.profdata %t-2.profdata +// RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw +// RUN: diff %t.normal.profdata %t-2.profdata // Online merging. // RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst index f81c80601e98a1..89b331c5bbd9c6 100644 --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -199,9 +199,10 @@ OPTIONS option to correlate the raw profile. .. option:: --binary-file= -Specify the executable that contains profile data and profile name sections for -the raw profile. When ``-profile-correlate=binary`` was used for -instrumentation, use this option to correlate the raw profile. + + Specify the executable that contains profile data and profile name sections for + the raw profile. When ``-profile-correlate=binary`` was used for + instrumentation, use this option to correlate the raw profile. .. option:: --temporal-profile-trace-reservoir-size diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index 661570eaee1130..ba82702354c189 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -5,7 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This file defines InstrProfCorrelator used to generate PGO/coverage profiles +// This file defines InstrProfCorrelator used to generate PGO/coverage profiles // from raw profile data and debug info/binary file. //===----------------------------------------------------------------------===// diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 168c2803145f1f..6796f2828d0172 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -206,8 +206,8 @@ InstrProfCorrelatorImpl::get( if (Obj.isELF() || Obj.isCOFF()) return std::make_unique>(std::move(Ctx)); return make_error( - instrprof_error::unable_to_correlate_profile, - "unsupported binary format (only ELF and COFF are supported)"); + instrprof_error::unable_to_correlate_profile, + "unsupported binary format (only ELF and COFF are supported)"); } template From 384ee6c1197683ff087bf04f8a5d1e79b27ecfc4 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 18 Oct 2023 17:05:44 -0400 Subject: [PATCH 05/10] add llvm-cov and IR test --- .../test/profile/instrprof-binary-correlate.c | 17 +++++++++++++++++ .../InstrProfiling/binary-correlate.ll | 11 +++++++++++ 2 files changed, 28 insertions(+) create mode 100644 llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c index d921c4c5dcb019..8009e572652689 100644 --- a/compiler-rt/test/profile/instrprof-binary-correlate.c +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -3,27 +3,44 @@ // RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal // RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report +// RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show // With -profile-correlate=binary flag // RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe // RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw +// RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report +// RUN: llvm-cov show --instr-profile=%t-1.profdata %t-1.exe > %t-1.show // RUN: diff %t.normal.profdata %t-1.profdata +// RUN: diff %t.normal.report %t-1.report +// RUN: diff %t.normal.show %t-1.show // Strip above binary and run // RUN: llvm-strip %t-1.exe -o %t-2.exe // RUN: env LLVM_PROFILE_FILE=%t-2.profraw %run %t-2.exe // RUN: llvm-profdata merge -o %t-2.profdata --binary-file=%t-1.exe %t-2.profraw +// RUN: llvm-cov report --instr-profile=%t-2.profdata %t-1.exe > %t-2.report +// RUN: llvm-cov show --instr-profile=%t-2.profdata %t-1.exe > %t-2.show // RUN: diff %t.normal.profdata %t-2.profdata +// RUN: diff %t.normal.report %t-2.report +// RUN: diff %t.normal.show %t-2.show // Online merging. // RUN: env LLVM_PROFILE_FILE=%t-3.profraw %run %t.normal // RUN: env LLVM_PROFILE_FILE=%t-4.profraw %run %t.normal // RUN: llvm-profdata merge -o %t.normal.merged.profdata %t-3.profraw %t-4.profraw +// RUN: llvm-cov report --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.report +// RUN: llvm-cov show --instr-profile=%t.normal.merged.profdata %t.normal > %t.normal.merged.show + // RUN: rm -rf %t.profdir && mkdir %t.profdir // RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe // RUN: env LLVM_PROFILE_FILE=%t.profdir/%m-4.profraw %run %t-2.exe // RUN: llvm-profdata merge -o %t-4.profdata --binary-file=%t-1.exe %t.profdir +// RUN: llvm-cov report --instr-profile=%t-4.profdata %t-1.exe > %t-4.report +// RUN: llvm-cov show --instr-profile=%t-4.profdata %t-1.exe > %t-4.show // RUN: diff %t.normal.merged.profdata %t-4.profdata +// RUN: diff %t.normal.merged.report %t-4.report +// RUN: diff %t.normal.merged.show %t-4.show // TODO: After adding support for binary ID, test binaries with different binary IDs. diff --git a/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll b/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll new file mode 100644 index 00000000000000..2206ce17d881df --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll @@ -0,0 +1,11 @@ +; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s + +; CHECK: @__profd_foo = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64) + +@__profn_foo = private constant [3 x i8] c"foo" +define void @_Z3foov() { + call void @llvm.instrprof.increment(ptr @__profn_foo, i64 12345678, i32 2, i32 0) + ret void +} + +declare void @llvm.instrprof.increment(ptr, i64, i32, i32) From 37f5fc4fb52eac17b01dced7cffeb6df1212d92c Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Thu, 2 Nov 2023 15:50:53 -0400 Subject: [PATCH 06/10] add missing DebugInfoCorrelate --- llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 6dcbf3ada5f0c7..1717f62327ea41 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -382,7 +382,7 @@ static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) { ProfileVersion |= VARIANT_MASK_CSIR_PROF; if (PGOInstrumentEntry) ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; - if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) + if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) ProfileVersion |= VARIANT_MASK_DBG_CORRELATE; if (PGOFunctionEntryCoverage) ProfileVersion |= From 8fb0a77739dc0b1920ac9510a5c4a7aae5e1009c Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 15 Nov 2023 17:09:26 -0500 Subject: [PATCH 07/10] Updated to use new section names `__llvm_covdata` and `__llvm_covnames` (or `.lprfcovdata` and `.lprfcovnames` on COFF) under binary correlation mode. The reason is that we don't want to rely on the bit in llvm profile version variable to indicate name/data sections are empty or not: https://github.com/llvm/llvm-project/pull/71996. And simply making __llvm_prf_{data}/{names} sections not allocated in ELF does not make `&__stop___llvm_prf_data - &__start___llvm_prf_data == 0`. It still returns the size of the non-allocated sections. So, using different section names eliminate the need to perform runtime check for data/name section size. With that, there is no need to have VARIANT_MASK_BIN_CORRELATE bit. Clang side codegen changes are not longer necessary. --- clang/lib/CodeGen/BackendUtil.cpp | 1 + clang/lib/CodeGen/CoverageMappingGen.cpp | 20 ------- .../CodeGen/coverage-profile-raw-version.c | 9 --- compiler-rt/include/profile/InstrProfData.inc | 19 ++++++- compiler-rt/lib/profile/InstrProfiling.h | 3 - .../profile/InstrProfilingPlatformWindows.c | 4 +- .../test/profile/instrprof-binary-correlate.c | 4 +- llvm/docs/CommandGuide/llvm-profdata.rst | 9 +-- llvm/include/llvm/ProfileData/InstrProf.h | 2 +- .../llvm/ProfileData/InstrProfData.inc | 19 ++++++- .../llvm/ProfileData/InstrProfReader.h | 17 ------ .../CodeGen/TargetLoweringObjectFileImpl.cpp | 27 ++------- .../Coverage/CoverageMappingReader.cpp | 56 +++++++++---------- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 54 +++++------------- llvm/lib/ProfileData/InstrProfReader.cpp | 9 +-- .../Instrumentation/InstrProfiling.cpp | 38 +++++++++---- .../InstrProfiling/binary-correlate.ll | 11 ---- .../InstrProfiling/coverage.ll | 8 +++ llvm/tools/llvm-profdata/llvm-profdata.cpp | 19 +++++++ 19 files changed, 150 insertions(+), 179 deletions(-) delete mode 100644 clang/test/CodeGen/coverage-profile-raw-version.c delete mode 100644 llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 7d86f85de7a2d9..713b1d9147f356 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -104,6 +104,7 @@ static cl::opt ClSanitizeOnOptimizerEarlyEP( cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); extern cl::opt ProfileCorrelate; + // Re-link builtin bitcodes after optimization static cl::opt ClRelinkBuiltinBitcodePostop( "relink-builtin-bitcode-postop", cl::Optional, diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index 57aec8499a8522..b16358ee117ae2 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -31,10 +31,6 @@ // is textually included. #define COVMAP_V3 -namespace llvm { -extern cl::opt ProfileCorrelate; -} // namespace llvm - static llvm::cl::opt EmptyLineCommentCoverage( "emptyline-comment-coverage", llvm::cl::desc("Emit emptylines and comment lines as skipped regions (only " @@ -1833,22 +1829,6 @@ void CoverageMappingModuleGen::emit() { llvm::GlobalValue::InternalLinkage, NamesArrVal, llvm::getCoverageUnusedNamesVarName()); } - const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); - llvm::Type *IntTy64 = llvm::Type::getInt64Ty(Ctx); - uint64_t ProfileVersion = INSTR_PROF_RAW_VERSION; - if (llvm::ProfileCorrelate == llvm::InstrProfCorrelator::BINARY) - ProfileVersion |= VARIANT_MASK_BIN_CORRELATE; - auto *VersionVariable = new llvm::GlobalVariable( - CGM.getModule(), llvm::Type::getInt64Ty(Ctx), true, - llvm::GlobalValue::WeakAnyLinkage, - llvm::Constant::getIntegerValue(IntTy64, llvm::APInt(64, ProfileVersion)), - VarName); - VersionVariable->setVisibility(llvm::GlobalValue::HiddenVisibility); - llvm::Triple TT(CGM.getModule().getTargetTriple()); - if (TT.supportsCOMDAT()) { - VersionVariable->setLinkage(llvm::GlobalValue::ExternalLinkage); - VersionVariable->setComdat(CGM.getModule().getOrInsertComdat(VarName)); - } } unsigned CoverageMappingModuleGen::getFileID(FileEntryRef File) { diff --git a/clang/test/CodeGen/coverage-profile-raw-version.c b/clang/test/CodeGen/coverage-profile-raw-version.c deleted file mode 100644 index 09e13d4a4ce63c..00000000000000 --- a/clang/test/CodeGen/coverage-profile-raw-version.c +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -mllvm -profile-correlate=binary -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -o - %s | FileCheck %s --check-prefix=BIN-CORRELATE - -// CHECK: @__llvm_profile_raw_version = {{.*}} i64 9 -// BIN-CORRELATE: @__llvm_profile_raw_version = {{.*}} i64 4294967305 - -int main() { - return 0; -} diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 7969207223c325..54d9186fd0a0dd 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -665,7 +671,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the 8th most significant bit * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. - * 1 in bit 32 indicates there is no profile name and data sections. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. @@ -675,7 +680,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) -#define VARIANT_MASK_BIN_CORRELATE (0x1ULL << 32) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) @@ -704,6 +708,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ @@ -716,6 +722,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lprfcovdata" +#define INSTR_PROF_COVNAME_COFF ".lprfcovnames" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -732,6 +743,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) @@ -757,6 +770,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Macros to define start/stop section symbol for a given * section on Linux. For instance diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index cffb128d5f0648..c5b0b34f2d8af0 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -269,9 +269,6 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); -/*! \brief Get the size of the profile name section in bytes. */ -uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); - /*! \brief Get the size in bytes of a single counter entry. */ size_t __llvm_profile_counter_entry_size(void); diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 1cecc26d034884..9dbd702865fd29 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,13 +13,13 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfc=.data") #pragma comment(linker, "/MERGE:.lprfb=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. - * Do *NOT* merge .lprfc and lprfd into .rdata. When binary profile correlation - * is enabled, llvm-profdata must be able to find after the fact. */ /* Allocate read-only section bounds. */ diff --git a/compiler-rt/test/profile/instrprof-binary-correlate.c b/compiler-rt/test/profile/instrprof-binary-correlate.c index 8009e572652689..8f421014cf5c9f 100644 --- a/compiler-rt/test/profile/instrprof-binary-correlate.c +++ b/compiler-rt/test/profile/instrprof-binary-correlate.c @@ -1,13 +1,13 @@ // REQUIRES: linux || windows // Default -// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang -o %t.normal -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal // RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw // RUN: llvm-cov report --instr-profile=%t.normal.profdata %t.normal > %t.normal.report // RUN: llvm-cov show --instr-profile=%t.normal.profdata %t.normal > %t.normal.show // With -profile-correlate=binary flag -// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary -fuse-ld=lld %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang -o %t-1.exe -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t-1.profraw %run %t-1.exe // RUN: llvm-profdata merge -o %t-1.profdata --binary-file=%t-1.exe %t-1.profraw // RUN: llvm-cov report --instr-profile=%t-1.profdata %t-1.exe > %t-1.report diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst index 89b331c5bbd9c6..f5e3c13ffbc8e6 100644 --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -195,8 +195,8 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-profile-correlate=debug-info`` was used for instrumentation, use this - option to correlate the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to correlate the raw profile. .. option:: --binary-file= @@ -352,8 +352,9 @@ OPTIONS .. option:: --debug-info= Specify the executable or ``.dSYM`` that contains debug info for the raw profile. - When ``-profile-correlate=debug-info`` was used for instrumentation, use this - option to show the correlated functions from the raw profile. + When ``--debug-info-correlate`` or ``--profile-correlate=debug-info`` was used + for instrumentation, use this option to show the correlated functions from the + raw profile. .. option:: --covered diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 5c7b57c09bb435..288dc71d756aee 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -462,7 +462,7 @@ class InstrProfSymtab { /// only initialize the symtab with reference to the data and /// the section base address. The decompression will be delayed /// until before it is used. See also \c create(StringRef) method. - Error create(object::SectionRef &Section, bool MightHasNullByte = false); + Error create(object::SectionRef &Section); /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 9c85dcac3445e0..54d9186fd0a0dd 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -295,6 +295,12 @@ INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -665,7 +671,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the 8th most significant bit * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. - * 1 in bit 32 indicates whether to use binary to correlate profiles. * 1 in bit 57 indicates there are context-sensitive records in the profile. * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. @@ -675,7 +680,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) -#define VARIANT_MASK_BIN_CORRELATE (0x1ULL << 32) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) @@ -704,6 +708,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ @@ -716,6 +722,11 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lprfcovdata" +#define INSTR_PROF_COVNAME_COFF ".lprfcovnames" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -732,6 +743,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) @@ -757,6 +770,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Macros to define start/stop section symbol for a given * section on Linux. For instance diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 609ac480b8df15..551e85a1237157 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -123,11 +123,6 @@ class InstrProfReader { virtual bool instrEntryBBEnabled() const = 0; - /// Return true if we must provide debug info to create PGO profiles. - virtual bool useDebugInfoCorrelate() const { return false; } - - virtual bool useBinaryCorrelate() const { return false; } - /// Return true if the profile has single byte counters representing coverage. virtual bool hasSingleByteCoverage() const = 0; @@ -384,18 +379,6 @@ class RawInstrProfReader : public InstrProfReader { return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; } - bool useDebugInfoCorrelate() const override { - return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; - } - - bool useBinaryCorrelate() const override { - return (Version & VARIANT_MASK_BIN_CORRELATE) != 0; - } - - bool useCorrelate() const { - return useDebugInfoCorrelate() || useBinaryCorrelate(); - } - bool hasSingleByteCoverage() const override { return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0; } diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 493dc58110e4fd..f0b249e410b973 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -57,7 +57,6 @@ #include "llvm/MC/MCValue.h" #include "llvm/MC/SectionKind.h" #include "llvm/ProfileData/InstrProf.h" -#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/Base64.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" @@ -72,10 +71,6 @@ using namespace llvm; using namespace dwarf; -namespace llvm { -extern cl::opt ProfileCorrelate; -} // namespace llvm - static cl::opt JumpTableInFunctionSection( "jumptable-in-function-section", cl::Hidden, cl::init(false), cl::desc("Putting Jump Table in function section")); @@ -477,15 +472,11 @@ static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) { /*AddSegmentInfo=*/false) || Name == getInstrProfSectionName(IPSK_covfun, Triple::ELF, /*AddSegmentInfo=*/false) || - Name == ".llvmbc" || Name == ".llvmcmd" || - // Instead of checking the flag, maybe we should check if - // VARIANT_MASK_BIN_CORRELATE flag is set at the profile version global - // variable. - (ProfileCorrelate == InstrProfCorrelator::BINARY && - (Name == getInstrProfSectionName(llvm::IPSK_name, Triple::ELF, - /*AddSegmentInfo=*/false) || - Name == getInstrProfSectionName(llvm::IPSK_data, Triple::ELF, - /*AddSegmentInfo=*/false)))) + Name == getInstrProfSectionName(IPSK_covdata, Triple::ELF, + /*AddSegmentInfo=*/false) || + Name == getInstrProfSectionName(IPSK_covname, Triple::ELF, + /*AddSegmentInfo=*/false) || + Name == ".llvmbc" || Name == ".llvmcmd") return SectionKind::getMetadata(); if (Name.empty() || Name[0] != '.') return K; @@ -1690,15 +1681,9 @@ static int getSelectionForCOFF(const GlobalValue *GV) { MCSection *TargetLoweringObjectFileCOFF::getExplicitSectionGlobal( const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const { - StringRef Name = GO->getSection(); - if (ProfileCorrelate == InstrProfCorrelator::BINARY && - (Name == getInstrProfSectionName(llvm::IPSK_name, Triple::COFF, - /*AddSegmentInfo=*/false) || - Name == getInstrProfSectionName(llvm::IPSK_data, Triple::COFF, - /*AddSegmentInfo=*/false))) - Kind = SectionKind::getMetadata(); int Selection = 0; unsigned Characteristics = getCOFFSectionFlags(Kind, TM); + StringRef Name = GO->getSection(); StringRef COMDATSymName = ""; if (GO->hasComdat()) { Selection = getSelectionForCOFF(GO); diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index f6e0efdc80617d..715f7c136ac467 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -457,7 +457,7 @@ Expected RawCoverageMappingDummyChecker::isDummy() { return Tag == Counter::Zero; } -Error InstrProfSymtab::create(SectionRef &Section, bool MightHasNullByte) { +Error InstrProfSymtab::create(SectionRef &Section) { Expected DataOrErr = Section.getContents(); if (!DataOrErr) return DataOrErr.takeError(); @@ -466,10 +466,13 @@ Error InstrProfSymtab::create(SectionRef &Section, bool MightHasNullByte) { // If this is a linked PE/COFF file, then we have to skip over the null byte // that is allocated in the .lprfn$A section in the LLVM profiling runtime. + // If the name section is .lprfcovnames, it doesn't have the null byte at the + // beginning. const ObjectFile *Obj = Section.getObject(); - if (MightHasNullByte && isa(Obj) && - !Obj->isRelocatableObject()) - Data = Data.drop_front(1); + if (isa(Obj) && !Obj->isRelocatableObject()) + if (Expected NameOrErr = Section.getName()) + if (*NameOrErr == getInstrProfSectionName(IPSK_name, Triple::COFF)) + Data = Data.drop_front(1); return Error::success(); } @@ -1054,11 +1057,30 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, // Look for the sections that we are interested in. auto ObjFormat = OF->getTripleObjectFormat(); + InstrProfSymtab ProfileNames; + std::vector NamesSectionRefs; + // If IPSK_name is not found, fallback to search for IPK_covname, which is + // used when binary correlation is enabled. auto NamesSection = lookupSections(*OF, getInstrProfSectionName(IPSK_name, ObjFormat, - /*AddSegmentInfo=*/false)); - if (auto E = NamesSection.takeError()) + /*AddSegmentInfo=*/false)); + if (auto E = NamesSection.takeError()) { + consumeError(std::move(E)); + NamesSection = + lookupSections(*OF, getInstrProfSectionName(IPSK_covname, ObjFormat, + /*AddSegmentInfo=*/false)); + if (auto E = NamesSection.takeError()) + return std::move(E); + } + NamesSectionRefs = *NamesSection; + + if (NamesSectionRefs.size() != 1) + return make_error( + coveragemap_error::malformed, + "the size of coverage mapping section is not one"); + if (Error E = ProfileNames.create(NamesSectionRefs.back())) return std::move(E); + auto CoverageSection = lookupSections(*OF, getInstrProfSectionName(IPSK_covmap, ObjFormat, /*AddSegmentInfo=*/false)); @@ -1073,28 +1095,6 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, return CoverageMappingOrErr.takeError(); StringRef CoverageMapping = CoverageMappingOrErr.get(); - InstrProfSymtab ProfileNames; - std::vector NamesSectionRefs = *NamesSection; - bool MightHasNullByte = true; - if (NamesSectionRefs.size() != 1) { - // By default, the profile name section in the binary starts with a null - // byte and followed by names. But if binary correlation is enabled, there - // will be 2 name sections. One contains only two null bytes, and another - // one contains the names without the dummy null byte at the beginning. - if (ObjFormat == Triple::COFF) { - NamesSectionRefs.erase( - std::remove_if(NamesSectionRefs.begin(), NamesSectionRefs.end(), - [](const SectionRef &S) { return S.isData(); })); - MightHasNullByte = false; - } - if (NamesSectionRefs.size() != 1) - return make_error( - coveragemap_error::malformed, - "the size of coverage mapping section is not one"); - } - if (Error E = ProfileNames.create(NamesSectionRefs.back(), MightHasNullByte)) - return std::move(E); - // Look for the coverage records section (Version4 only). auto CoverageRecordsSections = lookupSections(*OF, getInstrProfSectionName(IPSK_covfun, ObjFormat, diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index bd1e8c456f7c61..74fc89083ab39e 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -24,43 +24,17 @@ using namespace llvm; -namespace llvm { -cl::opt ProfileCorrelate( - "profile-correlate", - cl::desc("Use debug info or binary file to correlate profiles."), - cl::init(InstrProfCorrelator::NONE), - cl::values(clEnumValN(InstrProfCorrelator::NONE, "", - "No profile correlation"), - clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", - "Use debug info to correlate"), - clEnumValN(InstrProfCorrelator::BINARY, "binary", - "Use binary to correlate"))); -} // namespace llvm - /// Get profile section. Expected getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) { - // On COFF, the object file section name may end in "$M". This tells the - // linker to sort these sections between "$A" and "$Z". The linker removes the - // dollar and everything after it in the final binary. Do the same to match. Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); - auto StripSuffix = [ObjFormat](StringRef N) { - return ObjFormat == Triple::COFF ? N.split('$').first : N; - }; std::string ExpectedSectionName = getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); - ExpectedSectionName = StripSuffix(ExpectedSectionName); for (auto &Section : Obj.sections()) { if (auto SectionName = Section.getName()) - if (StripSuffix(*SectionName) == ExpectedSectionName) { - // There will be extra profile name and data sections in COFF used for - // runtime to find start/stop of those sections, skipping them. - // (details at compiler-rt/lib/profile/InstrProfilingPlatformWindows.c) - if (ObjFormat == Triple::COFF && IPSK != IPSK_cnts && Section.isData()) - continue; + if (*SectionName == ExpectedSectionName) return Section; - } } return make_error( instrprof_error::unable_to_correlate_profile, @@ -80,13 +54,13 @@ InstrProfCorrelator::Context::get(std::unique_ptr Buffer, if (auto Err = CountersSection.takeError()) return std::move(Err); if (FileKind == InstrProfCorrelator::BINARY) { - auto DataSection = getInstrProfSection(Obj, IPSK_data); + auto DataSection = getInstrProfSection(Obj, IPSK_covdata); if (auto Err = DataSection.takeError()) return std::move(Err); auto DataOrErr = DataSection->getContents(); if (!DataOrErr) return DataOrErr.takeError(); - auto NameSection = getInstrProfSection(Obj, IPSK_name); + auto NameSection = getInstrProfSection(Obj, IPSK_covname); if (auto Err = NameSection.takeError()) return std::move(Err); auto NameOrErr = NameSection->getContents(); @@ -103,7 +77,7 @@ InstrProfCorrelator::Context::get(std::unique_ptr Buffer, // In COFF object file, there's a null byte at the beginning of the counter // section which doesn't exist in raw profile. if (Obj.getTripleObjectFormat() == Triple::COFF) - C->CountersSectionStart++; + ++C->CountersSectionStart; C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; return Expected>(std::move(C)); @@ -461,27 +435,25 @@ Error DwarfInstrProfCorrelator::correlateProfileNameImpl() { template void BinaryInstrProfCorrelator::correlateProfileDataImpl( int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) { + using RawProfData = RawInstrProf::ProfileData; bool UnlimitedWarnings = (MaxWarnings == 0); // -N suppressed warnings means we can emit up to N (unsuppressed) warnings int NumSuppressedWarnings = -MaxWarnings; - const RawInstrProf::ProfileData *DataStart = - (const RawInstrProf::ProfileData *)this->Ctx->DataStart; - const RawInstrProf::ProfileData *DataEnd = - (const RawInstrProf::ProfileData *)this->Ctx->DataEnd; + const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart; + const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd; // We need to use < here because the last data record may have no padding. - for (const RawInstrProf::ProfileData *I = DataStart; I < DataEnd; - ++I) { + for (const RawProfData *I = DataStart; I < DataEnd; ++I) { uint64_t CounterPtr = this->template maybeSwap(I->CounterPtr); uint64_t CountersStart = this->Ctx->CountersSectionStart; uint64_t CountersEnd = this->Ctx->CountersSectionEnd; if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) { if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) { - WithColor::warning() << format( - "CounterPtr out of range for function: Actual=0x%x " - "Expected=[0x%x, 0x%x) at data offset=0x%x\n", - CounterPtr, CountersStart, CountersEnd, - (I - DataStart) * sizeof(RawInstrProf::ProfileData)); + WithColor::warning() + << format("CounterPtr out of range for function: Actual=0x%x " + "Expected=[0x%x, 0x%x) at data offset=0x%x\n", + CounterPtr, CountersStart, CountersEnd, + (I - DataStart) * sizeof(RawProfData)); } } // In binary correlation mode, the CounterPtr is an absolute address of the diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index de46d137bb8ca7..0753a80130ca26 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -566,10 +566,6 @@ Error RawInstrProfReader::readHeader( "\nPLEASE update this tool to version in the raw profile, or " "regenerate raw profile with expected version.") .str()); - if (useCorrelate() && !Correlator) - return error(instrprof_error::missing_correlation_info); - if (!useCorrelate() && Correlator) - return error(instrprof_error::unexpected_correlation_info); BinaryIdsSize = swap(Header.BinaryIdsSize); if (BinaryIdsSize % sizeof(uint64_t)) @@ -606,8 +602,9 @@ Error RawInstrProfReader::readHeader( if (Correlator) { // These sizes in the raw file are zero because we constructed them in the // Correlator. - assert(DataSize == 0 && NamesSize == 0); - assert(CountersDelta == 0 && NamesDelta == 0); + if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 && + NamesDelta == 0)) + return error(instrprof_error::unexpected_correlation_info); Data = Correlator->getDataPointer(); DataEnd = Data + Correlator->getDataSize(); NamesStart = Correlator->getNamesPointer(); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index d98c806ac6d889..7ad05073dba018 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -60,11 +60,24 @@ using namespace llvm; #define DEBUG_TYPE "instrprof" namespace llvm { -cl::opt - DebugInfoCorrelate("debug-info-correlate", - cl::desc("Use debug info to correlate profiles."), - cl::init(false)); -extern cl::opt ProfileCorrelate; +// TODO: Remove -debug-info-correlate in next LLVM release, in favor of +// -profile-correlate=debug-info. +cl::opt DebugInfoCorrelate( + "debug-info-correlate", + cl::desc("Use debug info to correlate profiles. (Deprecated, use " + "-profile-correlate=debug-info)"), + cl::init(false)); + +cl::opt ProfileCorrelate( + "profile-correlate", + cl::desc("Use debug info or binary file to correlate profiles."), + cl::init(InstrProfCorrelator::NONE), + cl::values(clEnumValN(InstrProfCorrelator::NONE, "", + "No profile correlation"), + clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", + "Use debug info to correlate"), + clEnumValN(InstrProfCorrelator::BINARY, "binary", + "Use binary to correlate"))); } // namespace llvm namespace { @@ -1345,16 +1358,18 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Constant *RelativeCounterPtr; GlobalVariable *BitmapPtr = PD.RegionBitmaps; Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); - // By default counter ptr and bitmap ptr are address relative to data section. - // When binary profile correlation is enabled, they should be absolute as data - // section will no longer load into memory. + InstrProfSectKind DataSectionKind; + // With binary profile correlation, profile data is not loaded into memory. + // profile data must reference profile counter with an absolute relocation. if (ProfileCorrelate == InstrProfCorrelator::BINARY) { + DataSectionKind = IPSK_covdata; RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy); if (BitmapPtr != nullptr) RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy); } else { // Reference the counter variable with a label difference (link-time // constant). + DataSectionKind = IPSK_data; RelativeCounterPtr = ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); @@ -1371,7 +1386,8 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Data->setInitializer(ConstantStruct::get(DataTy, DataVals)); Data->setVisibility(Visibility); - Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); + Data->setSection( + getInstrProfSectionName(DataSectionKind, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); maybeSetComdat(Data, Fn, CntsVarName); @@ -1457,7 +1473,9 @@ void InstrProfiling::emitNameData() { getInstrProfNamesVarName()); NamesSize = CompressedNameStr.size(); NamesVar->setSection( - getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); + ProfileCorrelate == InstrProfCorrelator::BINARY + ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat()) + : getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); // On COFF, it's important to reduce the alignment down to 1 to prevent the // linker from inserting padding before the start of the names section or // between names entries. diff --git a/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll b/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll deleted file mode 100644 index c7922fa45eca9a..00000000000000 --- a/llvm/test/Instrumentation/InstrProfiling/binary-correlate.ll +++ /dev/null @@ -1,11 +0,0 @@ -; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s - -; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64) - -@__profn_foo = private constant [3 x i8] c"foo" -define void @_Z3foov() { - call void @llvm.instrprof.increment(ptr @__profn_foo, i64 12345678, i32 2, i32 0) - ret void -} - -declare void @llvm.instrprof.increment(ptr, i64, i32, i32) diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll index 1401d8f620b3f4..bbf895ea4b34e1 100644 --- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll @@ -1,11 +1,19 @@ ; RUN: opt < %s -passes=instrprof -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -profile-correlate=binary -S | FileCheck %s --check-prefix=BINARY target triple = "aarch64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) +; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), @__profn_bar = private constant [3 x i8] c"bar" ; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 +; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) +; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), + +; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names" +; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames" define void @_Z3foov() { call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 1, i32 0) diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 27c67aa4345632..7dd030a60dfabf 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -125,6 +125,11 @@ cl::opt DebugInfoFilename( "the functions it found. For merge, use the provided debug info to " "correlate the raw profile."), cl::sub(ShowSubcommand), cl::sub(MergeSubcommand)); +cl::opt + BinaryFilename("binary-file", cl::init(""), + cl::desc("For merge, use the provided unstripped bianry to " + "correlate the raw profile."), + cl::sub(MergeSubcommand)); cl::opt FuncNameFilter( "function", cl::desc("Details for matching functions. For overlapping CSSPGO, this " @@ -788,6 +793,20 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, OutputFormat != PF_Text) exitWithError("unknown format is specified"); + // TODO: Maybe we should support correlation with mixture of different + // correlation modes(w/wo debug-info/object correlation). + if (!DebugInfoFilename.empty() && !BinaryFilename.empty()) + exitWithError("Expected only one of -debug-info, -binary-file"); + std::string CorrelateFilename; + ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE; + if (!DebugInfoFilename.empty()) { + CorrelateFilename = DebugInfoFilename; + CorrelateKind = ProfCorrelatorKind::DEBUG_INFO; + } else if (!BinaryFilename.empty()) { + CorrelateFilename = BinaryFilename; + CorrelateKind = ProfCorrelatorKind::BINARY; + } + std::unique_ptr Correlator; if (CorrelateKind != InstrProfCorrelator::NONE) { if (auto Err = InstrProfCorrelator::get(CorrelateFilename, CorrelateKind) From 1ee00530276db15ec3fd69b0a7503824204b580c Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 15 Nov 2023 17:29:41 -0500 Subject: [PATCH 08/10] fixup! format --- clang/lib/CodeGen/BackendUtil.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 713b1d9147f356..df280a982c6418 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -111,7 +111,6 @@ static cl::opt ClRelinkBuiltinBitcodePostop( cl::desc("Re-link builtin bitcodes after optimization."), cl::init(false)); } // namespace llvm - namespace { // Default filename used for profile generation. From f677688b462000d792c777b49a07ce8cd4bd7d1d Mon Sep 17 00:00:00 2001 From: My Name Date: Thu, 16 Nov 2023 12:11:03 -0500 Subject: [PATCH 09/10] Fix coff. Section names should be no greater than 8 bytes --- compiler-rt/include/profile/InstrProfData.inc | 4 +- .../profile/InstrProfilingPlatformWindows.c | 3 +- .../llvm/ProfileData/InstrProfData.inc | 4 +- .../Coverage/CoverageMappingReader.cpp | 37 +++++++++---------- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 7 ++++ 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 54d9186fd0a0dd..be2f502f60be0b 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -725,8 +725,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Since cov data and cov names sections are not allocated, we don't need to * access them at runtime. */ -#define INSTR_PROF_COVDATA_COFF ".lprfcovdata" -#define INSTR_PROF_COVNAME_COFF ".lprfcovnames" +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #ifdef _WIN32 /* Runtime section names and name strings. */ diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 9dbd702865fd29..9070b8a606eb54 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -13,13 +13,14 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") #pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. */ /* Allocate read-only section bounds. */ diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 54d9186fd0a0dd..be2f502f60be0b 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -725,8 +725,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Since cov data and cov names sections are not allocated, we don't need to * access them at runtime. */ -#define INSTR_PROF_COVDATA_COFF ".lprfcovdata" -#define INSTR_PROF_COVNAME_COFF ".lprfcovnames" +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #ifdef _WIN32 /* Runtime section names and name strings. */ diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 715f7c136ac467..9130b0397ebea6 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -471,7 +471,7 @@ Error InstrProfSymtab::create(SectionRef &Section) { const ObjectFile *Obj = Section.getObject(); if (isa(Obj) && !Obj->isRelocatableObject()) if (Expected NameOrErr = Section.getName()) - if (*NameOrErr == getInstrProfSectionName(IPSK_name, Triple::COFF)) + if (*NameOrErr != getInstrProfSectionName(IPSK_covname, Triple::COFF)) Data = Data.drop_front(1); return Error::success(); @@ -1001,10 +1001,13 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) { BytesInAddress, Endian, CompilationDir); } -/// Find all sections that match \p Name. There may be more than one if comdats -/// are in use, e.g. for the __llvm_covfun section on ELF. -static Expected> lookupSections(ObjectFile &OF, - StringRef Name) { +/// Find all sections that match \p IPSK name. There may be more than one if +/// comdats are in use, e.g. for the __llvm_covfun section on ELF. +static Expected> +lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) { + auto ObjFormat = OF.getTripleObjectFormat(); + auto Name = + getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); // On COFF, the object file section name may end in "$M". This tells the // linker to sort these sections between "$A" and "$Z". The linker removes the // dollar and everything after it in the final binary. Do the same to match. @@ -1019,8 +1022,13 @@ static Expected> lookupSections(ObjectFile &OF, Expected NameOrErr = Section.getName(); if (!NameOrErr) return NameOrErr.takeError(); - if (stripSuffix(*NameOrErr) == Name) + if (stripSuffix(*NameOrErr) == Name) { + // COFF profile name section contains two null bytes indicating the + // start/end of the section. If its size is 2 bytes, it's empty. + if (IsCOFF && IPSK == IPSK_name && Section.getSize() == 2) + continue; Sections.push_back(Section); + } } if (Sections.empty()) return make_error(coveragemap_error::no_data_found); @@ -1056,19 +1064,14 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, OF->isLittleEndian() ? llvm::endianness::little : llvm::endianness::big; // Look for the sections that we are interested in. - auto ObjFormat = OF->getTripleObjectFormat(); InstrProfSymtab ProfileNames; std::vector NamesSectionRefs; // If IPSK_name is not found, fallback to search for IPK_covname, which is // used when binary correlation is enabled. - auto NamesSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_name, ObjFormat, - /*AddSegmentInfo=*/false)); + auto NamesSection = lookupSections(*OF, IPSK_name); if (auto E = NamesSection.takeError()) { consumeError(std::move(E)); - NamesSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_covname, ObjFormat, - /*AddSegmentInfo=*/false)); + NamesSection = lookupSections(*OF, IPSK_covname); if (auto E = NamesSection.takeError()) return std::move(E); } @@ -1081,9 +1084,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, if (Error E = ProfileNames.create(NamesSectionRefs.back())) return std::move(E); - auto CoverageSection = - lookupSections(*OF, getInstrProfSectionName(IPSK_covmap, ObjFormat, - /*AddSegmentInfo=*/false)); + auto CoverageSection = lookupSections(*OF, IPSK_covmap); if (auto E = CoverageSection.takeError()) return std::move(E); std::vector CoverageSectionRefs = *CoverageSection; @@ -1096,9 +1097,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, StringRef CoverageMapping = CoverageMappingOrErr.get(); // Look for the coverage records section (Version4 only). - auto CoverageRecordsSections = - lookupSections(*OF, getInstrProfSectionName(IPSK_covfun, ObjFormat, - /*AddSegmentInfo=*/false)); + auto CoverageRecordsSections = lookupSections(*OF, IPSK_covfun); BinaryCoverageReader::FuncRecordsStorage FuncRecords; if (auto E = CoverageRecordsSections.takeError()) { diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp index 74fc89083ab39e..b18dcad6bd1c5c 100644 --- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -27,10 +27,17 @@ using namespace llvm; /// Get profile section. Expected getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) { + // On COFF, the getInstrProfSectionName returns the section names may followed + // by "$M". The linker removes the dollar and everything after it in the final + // binary. Do the same to match. Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat(); + auto StripSuffix = [ObjFormat](StringRef N) { + return ObjFormat == Triple::COFF ? N.split('$').first : N; + }; std::string ExpectedSectionName = getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false); + ExpectedSectionName = StripSuffix(ExpectedSectionName); for (auto &Section : Obj.sections()) { if (auto SectionName = Section.getName()) if (*SectionName == ExpectedSectionName) From 023a4a6ed35edc4dbfc4221bf297b952528225e7 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Tue, 5 Dec 2023 16:27:55 -0500 Subject: [PATCH 10/10] re-order macros in InstrProfData.inc so __llvm_cov* are together and reverted changes in tests that use debug info correlate --- compiler-rt/include/profile/InstrProfData.inc | 10 +++++----- .../profile/Darwin/instrprof-debug-info-correlate.c | 4 ++-- .../Linux/instrprof-debug-info-correlate-warnings.c | 2 +- .../profile/Linux/instrprof-debug-info-correlate.c | 6 +++--- .../Linux/instrprof-show-debug-info-correlation.c | 4 ++-- llvm/include/llvm/ProfileData/InstrProfData.inc | 10 +++++----- .../InstrProfiling/debug-info-correlate-coverage.ll | 2 +- .../InstrProfiling/debug-info-correlate.ll | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index be2f502f60be0b..0be6117b886a81 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -707,9 +707,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun -#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile #define INSTR_PROF_COVDATA_COMMON __llvm_covdata #define INSTR_PROF_COVNAME_COMMON __llvm_covnames +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ @@ -721,12 +721,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" -#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" /* Since cov data and cov names sections are not allocated, we don't need to * access them at runtime. */ #define INSTR_PROF_COVDATA_COFF ".lcovd" #define INSTR_PROF_COVNAME_COFF ".lcovn" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -742,9 +742,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF -#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) @@ -759,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) @@ -770,8 +772,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) -#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) -#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Macros to define start/stop section symbol for a given * section on Linux. For instance diff --git a/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c index 46d25a4e386dc3..f347d439e2e067 100644 --- a/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c +++ b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c @@ -1,5 +1,5 @@ // Value profiling is currently not supported in lightweight mode. -// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t.dSYM %t.proflite @@ -9,7 +9,7 @@ // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata) -// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov // RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov.dSYM %t.cov.proflite diff --git a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c index 25022f241a6d28..5069c6340b64fd 100644 --- a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c +++ b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-warnings.c @@ -1,6 +1,6 @@ // Disable full debug info and verify that we get warnings during merging -// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=2 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,LIMIT --implicit-check-not=warning // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=0 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,NOLIMIT --implicit-check-not=warning diff --git a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c index ccfd65b6f4c4b1..a918d7b6299005 100644 --- a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c +++ b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c @@ -3,19 +3,19 @@ // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal // RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw -// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.d4.proflite %run %t.d4 // RUN: llvm-profdata merge -o %t.d4.profdata --debug-info=%t.d4 %t.d4.proflite // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.d4.profdata) -// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t // RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite // RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata) -// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp +// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp // RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov // RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov %t.cov.proflite diff --git a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c index 93bf40f98d3ab6..245dc798910425 100644 --- a/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c +++ b/compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c @@ -1,8 +1,8 @@ -// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s +// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s // RUN: llvm-profdata show --debug-info=%t --detailed-summary --show-prof-sym-list | FileCheck %s // RUN: llvm-profdata show --debug-info=%t --show-format=yaml | FileCheck %s --match-full-lines --check-prefix YAML -// RUN: %clang_pgogen -o %t.no.dbg -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s +// RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s // RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG // NO-DBG: unable to correlate profile: could not find any profile data metadata in correlated file diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index be2f502f60be0b..0be6117b886a81 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -707,9 +707,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun -#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile #define INSTR_PROF_COVDATA_COMMON __llvm_covdata #define INSTR_PROF_COVNAME_COMMON __llvm_covnames +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ @@ -721,12 +721,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" -#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" /* Since cov data and cov names sections are not allocated, we don't need to * access them at runtime. */ #define INSTR_PROF_COVDATA_COFF ".lcovd" #define INSTR_PROF_COVNAME_COFF ".lcovn" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -742,9 +742,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF -#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) @@ -759,6 +759,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) @@ -770,8 +772,6 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) -#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) -#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Macros to define start/stop section symbol for a given * section on Linux. For instance diff --git a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll index dd646153381705..192bac6e503a07 100644 --- a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate-coverage.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=instrprof -profile-correlate=debug-info -S | opt -O2 -S | FileCheck %s +; RUN: opt < %s -passes=instrprof -debug-info-correlate -S | opt -O2 -S | FileCheck %s @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo diff --git a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll index 84eaab33701a48..fd868ead5b78d2 100644 --- a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll +++ b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=instrprof -profile-correlate=debug-info -S > %t.ll +; RUN: opt < %s -passes=instrprof -debug-info-correlate -S > %t.ll ; RUN: FileCheck < %t.ll --implicit-check-not "{{__llvm_prf_data|__llvm_prf_names}}" %s ; RUN: %llc_dwarf -O0 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s --check-prefix CHECK-DWARF