diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index f31a769b6b442..bd542bd0d2b2b 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -80,6 +80,10 @@ class MDBuilder { /// Return metadata containing the pseudo probe descriptor for a function. MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Function *F); + /// Return metadata containing llvm statistics. + MDNode * + createLLVMStats(ArrayRef> LLVMStatsVec); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h index 847b9ffc3cfc0..d743bc5371b58 100644 --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -180,6 +180,9 @@ class MCObjectFileInfo { MCSection *PseudoProbeSection = nullptr; MCSection *PseudoProbeDescSection = nullptr; + // Section for metadata of llvm statistics. + MCSection *LLVMStatsSection = nullptr; + // ELF specific sections. MCSection *DataRelROSection = nullptr; MCSection *MergeableConst4Section = nullptr; @@ -366,6 +369,8 @@ class MCObjectFileInfo { MCSection *getPseudoProbeDescSection(StringRef FuncName) const; + MCSection *getLLVMStatsSection() const; + MCSection *getPCSection(StringRef Name, const MCSection *TextSec) const; // ELF specific sections. diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 01f42477ad196..89148350c49aa 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -58,6 +58,7 @@ #include "llvm/MC/MCValue.h" #include "llvm/MC/SectionKind.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/Base64.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" @@ -366,6 +367,31 @@ void TargetLoweringObjectFileELF::emitModuleMetadata(MCStreamer &Streamer, } } + if (NamedMDNode *LLVMStats = M.getNamedMetadata("llvm.stats")) { + // Emit the metadata for llvm statistics into .llvm_stats section, which is + // formatted as a list of key/value pair, the value is base64 encoded. + auto *S = C.getObjectFileInfo()->getLLVMStatsSection(); + Streamer.switchSection(S); + for (const auto *Operand : LLVMStats->operands()) { + const auto *MD = cast(Operand); + assert(MD->getNumOperands() % 2 == 0 && + ("Operand num should be even for a list of key/value pair")); + for (size_t I = 0; I < MD->getNumOperands(); I += 2) { + // Encode the key string size. + auto *Key = cast(MD->getOperand(I)); + Streamer.emitULEB128IntValue(Key->getString().size()); + Streamer.emitBytes(Key->getString()); + // Encode the value into a Base64 string. + std::string Value = encodeBase64( + Twine(mdconst::dyn_extract(MD->getOperand(I + 1)) + ->getZExtValue()) + .str()); + Streamer.emitULEB128IntValue(Value.size()); + Streamer.emitBytes(Value); + } + } + } + unsigned Version = 0; unsigned Flags = 0; StringRef Section; diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index d028eb4a21dd9..df08eb2063efb 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -344,3 +344,15 @@ MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Ops[2] = createString(F->getName()); return MDNode::get(Context, Ops); } + +MDNode * +MDBuilder::createLLVMStats(ArrayRef> LLVMStats) { + auto *Int64Ty = Type::getInt64Ty(Context); + SmallVector Ops(LLVMStats.size() * 2); + for (size_t I = 0; I < LLVMStats.size(); I++) { + Ops[I * 2] = createString(LLVMStats[I].first); + Ops[I * 2 + 1] = + createConstant(ConstantInt::get(Int64Ty, LLVMStats[I].second)); + } + return MDNode::get(Context, Ops); +} diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index b5460ced752a6..57839473205f9 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -531,6 +531,8 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) { PseudoProbeSection = Ctx->getELFSection(".pseudo_probe", DebugSecType, 0); PseudoProbeDescSection = Ctx->getELFSection(".pseudo_probe_desc", DebugSecType, 0); + + LLVMStatsSection = Ctx->getELFSection(".llvm_stats", ELF::SHT_PROGBITS, 0); } void MCObjectFileInfo::initGOFFMCObjectFileInfo(const Triple &T) { @@ -1199,6 +1201,10 @@ MCObjectFileInfo::getPseudoProbeDescSection(StringRef FuncName) const { return PseudoProbeDescSection; } +MCSection *MCObjectFileInfo::getLLVMStatsSection() const { + return LLVMStatsSection; +} + MCSection *MCObjectFileInfo::getPCSection(StringRef Name, const MCSection *TextSec) const { if (Ctx->getObjectFileType() != MCContext::IsELF) diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index f01a3ef867619..66c1e5b4bd594 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -133,6 +133,11 @@ static cl::opt ReportProfileStaleness( "report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics.")); +static cl::opt PersistProfileStaleness( + "persist-profile-staleness", cl::Hidden, cl::init(false), + cl::desc("Compute stale profile statistical metrics and write it into the " + "native object file(.llvm_stats section).")); + static cl::opt ProfileSampleAccurate( "profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " @@ -2041,7 +2046,7 @@ bool SampleProfileLoader::doInitialization(Module &M, } } - if (ReportProfileStaleness) { + if (ReportProfileStaleness || PersistProfileStaleness) { MatchingManager = std::make_unique(M, *Reader, ProbeManager.get()); } @@ -2150,18 +2155,42 @@ void SampleProfileMatcher::detectProfileMismatch() { detectProfileMismatch(F, *FS); } - if (FunctionSamples::ProfileIsProbeBased) { - errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")" - << " of functions' profile are invalid and " - << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples + if (ReportProfileStaleness) { + if (FunctionSamples::ProfileIsProbeBased) { + errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")" + << " of functions' profile are invalid and " + << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples + << ")" + << " of samples are discarded due to function hash mismatch.\n"; + } + errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite << ")" - << " of samples are discarded due to function hash mismatch.\n"; + << " of callsites' profile are invalid and " + << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples + << ")" + << " of samples are discarded due to callsite location mismatch.\n"; + } + + if (PersistProfileStaleness) { + LLVMContext &Ctx = M.getContext(); + MDBuilder MDB(Ctx); + + SmallVector> ProfStatsVec; + if (FunctionSamples::ProfileIsProbeBased) { + ProfStatsVec.emplace_back("NumMismatchedFuncHash", NumMismatchedFuncHash); + ProfStatsVec.emplace_back("TotalProfiledFunc", TotalProfiledFunc); + ProfStatsVec.emplace_back("MismatchedFuncHashSamples", + MismatchedFuncHashSamples); + ProfStatsVec.emplace_back("TotalFuncHashSamples", TotalFuncHashSamples); + } + ProfStatsVec.emplace_back("MismatchedCallsiteSamples", + MismatchedCallsiteSamples); + ProfStatsVec.emplace_back("TotalCallsiteSamples", TotalCallsiteSamples); + + auto *MD = MDB.createLLVMStats(ProfStatsVec); + auto *NMD = M.getOrInsertNamedMetadata("llvm.stats"); + NMD->addOperand(MD); } - errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite << ")" - << " of callsites' profile are invalid and " - << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples - << ")" - << " of samples are discarded due to callsite location mismatch.\n"; } bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -2208,7 +2237,7 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, assert(SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"); - if (ReportProfileStaleness) + if (ReportProfileStaleness || PersistProfileStaleness) MatchingManager->detectProfileMismatch(); bool retval = false; diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll index 68f8e19b10701..413c7120dcf6e 100644 --- a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll +++ b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll @@ -1,8 +1,15 @@ -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -S 2>%t +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll ; RUN: FileCheck %s --input-file %t +; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD +; RUN: llc < %t.ll -filetype=obj -o %t.obj +; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ ; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch. +; CHECK-MD: ![[#]] = !{!"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30} + +; CHECK-OBJ: .llvm_stats + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll index 0e88dea3cdd02..c2c6dfa9c0f28 100644 --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll @@ -1,9 +1,16 @@ -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -S 2>%t +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll ; RUN: FileCheck %s --input-file %t +; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD +; RUN: llc < %t.ll -filetype=obj -o %t.obj +; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ ; CHECK: (1/3) of functions' profile are invalid and (10/50) of samples are discarded due to function hash mismatch. ; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch. +; CHECK-MD: ![[#]] = !{!"NumMismatchedFuncHash", i64 1, !"TotalProfiledFunc", i64 3, !"MismatchedFuncHashSamples", i64 10, !"TotalFuncHashSamples", i64 50, !"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30} + +; CHECK-OBJ: .llvm_stats + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"