From 259e4c565846eea4fa09941f1a776636d4d024bc Mon Sep 17 00:00:00 2001 From: Hongtao Yu Date: Wed, 27 Oct 2021 16:56:06 -0700 Subject: [PATCH] [CSSPGO] Trim cold base profiles for the CS preinliner. Adding support to the CS preinliner to trim cold base profiles. This makes trimming consistent with the inline decision made by the preinliner. Also disable the existing profile merger when preinliner is on unless explicitly specified. Reviewed By: wenlei, wlei Differential Revision: https://reviews.llvm.org/D112489 --- llvm/include/llvm/ProfileData/SampleProf.h | 11 +++++++++-- llvm/lib/ProfileData/SampleProf.cpp | 15 +++++++++++---- llvm/test/tools/llvm-profgen/cs-preinline.test | 10 ++++++++++ llvm/tools/llvm-profdata/llvm-profdata.cpp | 2 +- llvm/tools/llvm-profgen/CSPreInliner.cpp | 2 +- llvm/tools/llvm-profgen/ProfileGenerator.cpp | 11 ++++++----- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 89faa835a15ea..c61e334405c63 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -1128,11 +1128,18 @@ template class SampleSorter { class SampleContextTrimmer { public: SampleContextTrimmer(SampleProfileMap &Profiles) : ProfileMap(Profiles){}; - // Trim and merge cold context profile when requested. + // Trim and merge cold context profile when requested. TrimBaseProfileOnly + // should only be effective when TrimColdContext is true. On top of + // TrimColdContext, TrimBaseProfileOnly can be used to specify to trim all + // cold profiles or only cold base profiles. Trimming base profiles only is + // mainly to honor the preinliner decsion. Note that when MergeColdContext is + // true, preinliner decsion is not honored anyway so TrimBaseProfileOnly will + // be ignored. void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, - uint32_t ColdContextFrameLength); + uint32_t ColdContextFrameLength, + bool TrimBaseProfileOnly); // Canonicalize context profile name and attributes. void canonicalizeContextProfiles(); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index 07af3f2d6722e..cb5dd92913205 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -331,7 +331,7 @@ std::error_code ProfileSymbolList::read(const uint8_t *Data, void SampleContextTrimmer::trimAndMergeColdContextProfiles( uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, - uint32_t ColdContextFrameLength) { + uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly) { if (!TrimColdContext && !MergeColdContext) return; @@ -339,14 +339,21 @@ void SampleContextTrimmer::trimAndMergeColdContextProfiles( if (ColdCountThreshold == 0) return; + // Trimming base profiles only is mainly to honor the preinliner decsion. When + // MergeColdContext is true preinliner decsion is not honored anyway so turn + // off TrimBaseProfileOnly. + if (MergeColdContext) + TrimBaseProfileOnly = false; + // Filter the cold profiles from ProfileMap and move them into a tmp // container std::vector> ColdProfiles; for (const auto &I : ProfileMap) { + const SampleContext &Context = I.first; const FunctionSamples &FunctionProfile = I.second; - if (FunctionProfile.getTotalSamples() >= ColdCountThreshold) - continue; - ColdProfiles.emplace_back(I.first, &I.second); + if (FunctionProfile.getTotalSamples() < ColdCountThreshold && + (!TrimBaseProfileOnly || Context.isBaseContext())) + ColdProfiles.emplace_back(Context, &I.second); } // Remove the cold profile from ProfileMap and merge them into diff --git a/llvm/test/tools/llvm-profgen/cs-preinline.test b/llvm/test/tools/llvm-profgen/cs-preinline.test index beca0d08f5160..eda62e891cc0f 100644 --- a/llvm/test/tools/llvm-profgen/cs-preinline.test +++ b/llvm/test/tools/llvm-profgen/cs-preinline.test @@ -10,6 +10,11 @@ ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --output=%t --csspgo-preinliner=1 -sample-profile-cold-inline-threshold=0 ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-NO-PREINL +; Test cold profile trimming. Only base profiles should be dropped. +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --output=%t --csspgo-preinliner=1 --csprof-trim-cold-context=1 --profile-summary-hot-count=250 + +; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-TRIM + ; CHECK-DEFAULT: [main:1 @ foo]:225:0 ; CHECK-DEFAULT-NEXT: 2.1: 14 ; CHECK-DEFAULT-NEXT: 3: 15 @@ -39,3 +44,8 @@ ; CHECK-NO-PREINL-NEXT:[bar]:84:0 ; CHECK-NO-PREINL-NEXT: 1: 14 ; CHECK-NO-PREINL-NEXT: !Attributes: 1 + +; CHECK-TRIM-NOT: [foo]:225:0 +; CHECK-TRIM:[foo:3.1 @ bar]:84:0 +; CHECK-TRIM-NEXT: 1: 14 +; CHECK-TRIM-NEXT: !Attributes: 3 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 8994b566862b1..fd67cac3cdd2a 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -759,7 +759,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, SampleContextTrimmer(ProfileMap) .trimAndMergeColdContextProfiles( SampleProfColdThreshold, SampleTrimColdContext, - SampleMergeColdContext, SampleColdContextFrameDepth); + SampleMergeColdContext, SampleColdContextFrameDepth, false); } auto WriterOrErr = diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp index 06c3a41ad14ad..1928da863606f 100644 --- a/llvm/tools/llvm-profgen/CSPreInliner.cpp +++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp @@ -252,7 +252,7 @@ void CSPreInliner::run() { // trim out such profiles from the output. std::vector ProfilesToBeRemoved; for (auto &It : ProfileMap) { - SampleContext Context = It.second.getContext(); + SampleContext &Context = It.second.getContext(); if (!Context.isBaseContext() && !Context.hasState(InlinedContext)) { assert(Context.hasState(MergedContext) && "Not inlined context profile should be merged already"); diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp index 6b0a64cf23afb..ac8cdec66eb32 100644 --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -657,16 +657,17 @@ void CSProfileGenerator::postProcessProfiles() { if (EnableCSPreInliner) { CSPreInliner(ProfileMap, *Binary, HotCountThreshold, ColdCountThreshold) .run(); + // Turn off the profile merger by default unless it is explicitly enabled. + if (!CSProfMergeColdContext.getNumOccurrences()) + CSProfMergeColdContext = false; } - // Trim and merge cold context profile using cold threshold above. By default, - // we skip such merging and trimming when preinliner is on. - if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() || - CSProfMergeColdContext.getNumOccurrences()) { + // Trim and merge cold context profile using cold threshold above. + if (CSProfTrimColdContext || CSProfMergeColdContext) { SampleContextTrimmer(ProfileMap) .trimAndMergeColdContextProfiles( HotCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, - CSProfMaxColdContextDepth); + CSProfMaxColdContextDepth, EnableCSPreInliner); } }