diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 936ebcecfe96d..55418d9d0f9cb 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -120,6 +120,7 @@ enum SecType { SecProfSummary = 1, SecNameTable = 2, SecProfileSymbolList = 3, + SecFuncOffsetTable = 4, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -135,6 +136,8 @@ static inline std::string getSecName(SecType Type) { return "NameTableSection"; case SecProfileSymbolList: return "ProfileSymbolListSection"; + case SecFuncOffsetTable: + return "FuncOffsetTableSection"; case SecLBRProfile: return "LBRProfileSection"; } diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index 424818bbb26df..6f6482747fbc1 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -279,7 +279,7 @@ class SampleProfileReader { /// Print the profile for \p FName on stream \p OS. void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); - virtual void collectFuncsToUse(const Module &M) {} + virtual void collectFuncsFrom(const Module &M) {} /// Print all the profiles on stream \p OS. void dump(raw_ostream &OS = dbgs()); @@ -424,7 +424,7 @@ class SampleProfileReaderBinary : public SampleProfileReader { bool at_eof() const { return Data >= End; } /// Read the next function profile instance. - std::error_code readFuncProfile(); + std::error_code readFuncProfile(const uint8_t *Start); /// Read the contents of the given profile instance. std::error_code readProfile(FunctionSamples &FProfile); @@ -526,7 +526,17 @@ class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { virtual std::error_code verifySPMagic(uint64_t Magic) override; virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, SecType Type) override; - std::error_code readProfileSymbolList(uint64_t Size); + std::error_code readProfileSymbolList(); + std::error_code readFuncOffsetTable(); + std::error_code readFuncProfiles(); + + /// The table mapping from function name to the offset of its FunctionSample + /// towards file start. + DenseMap FuncOffsetTable; + /// The set containing the functions to use when compiling a module. + DenseSet FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; public: SampleProfileReaderExtBinary(std::unique_ptr B, LLVMContext &C, @@ -539,6 +549,9 @@ class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { virtual std::unique_ptr getProfileSymbolList() override { return std::move(ProfSymList); }; + + /// Collect functions with definitions in Module \p M. + void collectFuncsFrom(const Module &M) override; }; class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { @@ -571,7 +584,7 @@ class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { std::error_code read() override; /// Collect functions to be used when compiling Module \p M. - void collectFuncsToUse(const Module &M) override; + void collectFuncsFrom(const Module &M) override; }; using InlineCallStack = SmallVector; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index ce60baf66c65c..cc951594c9e24 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -153,14 +153,15 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { protected: uint64_t markSectionStart(SecType Type); std::error_code addNewSection(SecType Sec, uint64_t SectionStart); - virtual void initSectionLayout() = 0; + virtual void initSectionHdrLayout() = 0; virtual std::error_code writeSections(const StringMap &ProfileMap) = 0; - // Specifiy the section layout in the profile. Note that the order in - // SecHdrTable (order to collect sections) may be different from the - // order in SectionLayout (order to write out sections into profile). - SmallVector SectionLayout; + // Specifiy the order of sections in section header table. Note + // the order of sections in the profile may be different that the + // order in SectionHdrLayout. sample Reader will follow the order + // in SectionHdrLayout to read each section. + SmallVector SectionHdrLayout; private: void allocSecHdrTable(); @@ -193,23 +194,44 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { public: SampleProfileWriterExtBinary(std::unique_ptr &OS) : SampleProfileWriterExtBinaryBase(OS) { - initSectionLayout(); + initSectionHdrLayout(); } + virtual std::error_code writeSample(const FunctionSamples &S) override; virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { ProfSymList = PSL; }; private: - virtual void initSectionLayout() override { - SectionLayout = {{SecProfSummary, 0, 0, 0}, - {SecNameTable, 0, 0, 0}, - {SecLBRProfile, 0, 0, 0}, - {SecProfileSymbolList, 0, 0, 0}}; + virtual void initSectionHdrLayout() override { + // Note that SecFuncOffsetTable section is written after SecLBRProfile + // in the profile, but is put before SecLBRProfile in SectionHdrLayout. + // + // This is because sample reader follows the order of SectionHdrLayout to + // read each section, to read function profiles on demand sample reader + // need to get the offset of each function profile first. + // + // SecFuncOffsetTable section is written after SecLBRProfile in the + // profile because FuncOffsetTable needs to be populated while section + // SecLBRProfile is written. + SectionHdrLayout = {{SecProfSummary, 0, 0, 0}, + {SecNameTable, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0}}; }; virtual std::error_code writeSections(const StringMap &ProfileMap) override; ProfileSymbolList *ProfSymList = nullptr; + + // Save the start of SecLBRProfile so we can compute the offset to the + // start of SecLBRProfile for each Function's Profile and will keep it + // in FuncOffsetTable. + uint64_t SecLBRProfileStart; + // FuncOffsetTable maps function name to its profile offset in SecLBRProfile + // section. It is used to load function profile on demand. + MapVector FuncOffsetTable; + std::error_code writeFuncOffsetTable(); }; // CompactBinary is a compact format of binary profile which both reduces diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index 6d00404b0bb22..cf3e56728e235 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -439,7 +439,9 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { return sampleprof_error::success; } -std::error_code SampleProfileReaderBinary::readFuncProfile() { +std::error_code +SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { + Data = Start; auto NumHeadSamples = readNumber(); if (std::error_code EC = NumHeadSamples.getError()) return EC; @@ -461,7 +463,7 @@ std::error_code SampleProfileReaderBinary::readFuncProfile() { std::error_code SampleProfileReaderBinary::read() { while (!at_eof()) { - if (std::error_code EC = readFuncProfile()) + if (std::error_code EC = readFuncProfile(Data)) return EC; } @@ -483,13 +485,15 @@ SampleProfileReaderExtBinary::readOneSection(const uint8_t *Start, return EC; break; case SecLBRProfile: - while (Data < Start + Size) { - if (std::error_code EC = readFuncProfile()) - return EC; - } + if (std::error_code EC = readFuncProfiles()) + return EC; break; case SecProfileSymbolList: - if (std::error_code EC = readProfileSymbolList(Size)) + if (std::error_code EC = readProfileSymbolList()) + return EC; + break; + case SecFuncOffsetTable: + if (std::error_code EC = readFuncOffsetTable()) return EC; break; default: @@ -498,15 +502,65 @@ SampleProfileReaderExtBinary::readOneSection(const uint8_t *Start, return sampleprof_error::success; } -std::error_code -SampleProfileReaderExtBinary::readProfileSymbolList(uint64_t Size) { +void SampleProfileReaderExtBinary::collectFuncsFrom(const Module &M) { + UseAllFuncs = false; + FuncsToUse.clear(); + for (auto &F : M) + FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); +} + +std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + FuncOffsetTable.reserve(*Size); + for (uint32_t I = 0; I < *Size; ++I) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + + auto Offset = readNumber(); + if (std::error_code EC = Offset.getError()) + return EC; + + FuncOffsetTable[*FName] = *Offset; + } + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderExtBinary::readFuncProfiles() { + const uint8_t *Start = Data; + if (UseAllFuncs) { + while (Data < End) { + if (std::error_code EC = readFuncProfile(Data)) + return EC; + } + assert(Data == End && "More data is read than expected"); + return sampleprof_error::success; + } + + for (auto Name : FuncsToUse) { + auto iter = FuncOffsetTable.find(Name); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } + Data = End; + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderExtBinary::readProfileSymbolList() { if (!ProfSymList) ProfSymList = std::make_unique(); - if (std::error_code EC = ProfSymList->read(Data, Size)) + if (std::error_code EC = ProfSymList->read(Data, End - Data)) return EC; - Data = Data + Size; + Data = End; return sampleprof_error::success; } @@ -600,9 +654,9 @@ std::error_code SampleProfileReaderCompactBinary::read() { for (auto Offset : OffsetsToUse) { const uint8_t *SavedData = Data; - Data = reinterpret_cast(Buffer->getBufferStart()) + - Offset; - if (std::error_code EC = readFuncProfile()) + if (std::error_code EC = readFuncProfile( + reinterpret_cast(Buffer->getBufferStart()) + + Offset)) return EC; Data = SavedData; } @@ -719,8 +773,16 @@ uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) { } uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { - auto &LastEntry = SecHdrTable.back(); - return LastEntry.Offset + LastEntry.Size; + // Sections in SecHdrTable is not necessarily in the same order as + // sections in the profile because section like FuncOffsetTable needs + // to be written after section LBRProfile but needs to be read before + // section LBRProfile, so we cannot simply use the last entry in + // SecHdrTable to calculate the file size. + uint64_t FileSize = 0; + for (auto &Entry : SecHdrTable) { + FileSize = std::max(Entry.Offset + Entry.Size, FileSize); + } + return FileSize; } bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { @@ -812,13 +874,11 @@ std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { return sampleprof_error::success; } -void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) { +void SampleProfileReaderCompactBinary::collectFuncsFrom(const Module &M) { UseAllFuncs = false; FuncsToUse.clear(); - for (auto &F : M) { - StringRef CanonName = FunctionSamples::getCanonicalFnName(F); - FuncsToUse.insert(CanonName); - } + for (auto &F : M) + FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); } std::error_code SampleProfileReaderBinary::readSummaryEntry( diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index 03446367665bd..8d09af31f94bf 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -76,7 +76,7 @@ SampleProfileWriter::write(const StringMap &ProfileMap) { SecHdrTableEntry & SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type) { auto SecIt = std::find_if( - SectionLayout.begin(), SectionLayout.end(), + SectionHdrLayout.begin(), SectionHdrLayout.end(), [=](const auto &Entry) -> bool { return Entry.Type == Type; }); return *SecIt; } @@ -143,6 +143,29 @@ std::error_code SampleProfileWriterExtBinaryBase::write( return sampleprof_error::success; } +std::error_code +SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) { + uint64_t Offset = OutputStream->tell(); + StringRef Name = S.getName(); + FuncOffsetTable[Name] = Offset - SecLBRProfileStart; + encodeULEB128(S.getHeadSamples(), *OutputStream); + return writeBody(S); +} + +std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() { + auto &OS = *OutputStream; + + // Write out the table size. + encodeULEB128(FuncOffsetTable.size(), OS); + + // Write out FuncOffsetTable. + for (auto entry : FuncOffsetTable) { + writeNameIdx(entry.first); + encodeULEB128(entry.second, OS); + } + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinary::writeSections( const StringMap &ProfileMap) { uint64_t SectionStart = markSectionStart(SecProfSummary); @@ -163,6 +186,7 @@ std::error_code SampleProfileWriterExtBinary::writeSections( return EC; SectionStart = markSectionStart(SecLBRProfile); + SecLBRProfileStart = OutputStream->tell(); if (std::error_code EC = writeFuncProfiles(ProfileMap)) return EC; if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart)) @@ -178,6 +202,12 @@ std::error_code SampleProfileWriterExtBinary::writeSections( if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart)) return EC; + SectionStart = markSectionStart(SecFuncOffsetTable); + if (std::error_code EC = writeFuncOffsetTable()) + return EC; + if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart)) + return EC; + return sampleprof_error::success; } @@ -359,7 +389,7 @@ std::error_code SampleProfileWriterBinary::writeHeader( } void SampleProfileWriterExtBinaryBase::setToCompressAllSections() { - for (auto &Entry : SectionLayout) + for (auto &Entry : SectionHdrLayout) addSecFlags(Entry, SecFlagCompress); } @@ -369,7 +399,7 @@ void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) { void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type, SecFlags Flags) { - for (auto &Entry : SectionLayout) { + for (auto &Entry : SectionHdrLayout) { if (Entry.Type == Type) addSecFlags(Entry, Flags); } @@ -378,9 +408,9 @@ void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type, void SampleProfileWriterExtBinaryBase::allocSecHdrTable() { support::endian::Writer Writer(*OutputStream, support::little); - Writer.write(static_cast(SectionLayout.size())); + Writer.write(static_cast(SectionHdrLayout.size())); SecHdrTableOffset = OutputStream->tell(); - for (uint32_t i = 0; i < SectionLayout.size(); i++) { + for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) { Writer.write(static_cast(-1)); Writer.write(static_cast(-1)); Writer.write(static_cast(-1)); @@ -402,14 +432,15 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() { IndexMap.insert({static_cast(SecHdrTable[i].Type), i}); } - // Write the sections in the order specified in SectionLayout. - // That is the sections order Reader will see. Note that the - // sections order in which Reader expects to read may be different - // from the order in which Writer is able to write, so we need - // to adjust the order in SecHdrTable to be consistent with - // SectionLayout when we write SecHdrTable to the memory. - for (uint32_t i = 0; i < SectionLayout.size(); i++) { - uint32_t idx = IndexMap[static_cast(SectionLayout[i].Type)]; + // Write the section header table in the order specified in + // SectionHdrLayout. That is the sections order Reader will see. + // Note that the sections order in which Reader expects to read + // may be different from the order in which Writer is able to + // write, so we need to adjust the order in SecHdrTable to be + // consistent with SectionHdrLayout when we write SecHdrTable + // to the memory. + for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) { + uint32_t idx = IndexMap[static_cast(SectionHdrLayout[i].Type)]; Writer.write(static_cast(SecHdrTable[idx].Type)); Writer.write(static_cast(SecHdrTable[idx].Flags)); Writer.write(static_cast(SecHdrTable[idx].Offset)); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index c9e7a19c380a2..aa22ac3b449c3 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1682,7 +1682,7 @@ bool SampleProfileLoader::doInitialization(Module &M) { return false; } Reader = std::move(ReaderOrErr.get()); - Reader->collectFuncsToUse(M); + Reader->collectFuncsFrom(M); ProfileIsValid = (Reader->read() == sampleprof_error::success); PSL = Reader->getProfileSymbolList(); diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/inline.extbinary.afdo index f1a1f87820f61..4a5e0cc527878 100644 Binary files a/llvm/test/Transforms/SampleProfile/Inputs/inline.extbinary.afdo and b/llvm/test/Transforms/SampleProfile/Inputs/inline.extbinary.afdo differ diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo index c05af6fb4a6dc..137f4c92f04c7 100644 Binary files a/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo and b/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo differ diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp index dd5ded5bc9b2f..59ed19d292eff 100644 --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -54,7 +54,7 @@ struct SampleProfTest : ::testing::Test { auto ReaderOrErr = SampleProfileReader::create(Profile, Context); ASSERT_TRUE(NoError(ReaderOrErr.getError())); Reader = std::move(ReaderOrErr.get()); - Reader->collectFuncsToUse(M); + Reader->collectFuncsFrom(M); } void testRoundTrip(SampleProfileFormat Format, bool Remap) { @@ -86,6 +86,13 @@ struct SampleProfTest : ::testing::Test { BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000); BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437); + StringRef BazName("_Z3bazi"); + FunctionSamples BazSamples; + BazSamples.setName(BazName); + BazSamples.addTotalSamples(12557); + BazSamples.addHeadSamples(1257); + BazSamples.addBodySamples(1, 0, 12557); + Module M("my_module", Context); FunctionType *fn_type = FunctionType::get(Type::getVoidTy(Context), {}, false); @@ -95,6 +102,7 @@ struct SampleProfTest : ::testing::Test { StringMap Profiles; Profiles[FooName] = std::move(FooSamples); Profiles[BarName] = std::move(BarSamples); + Profiles[BazName] = std::move(BazSamples); ProfileSymbolList List; if (Format == SampleProfileFormat::SPF_Ext_Binary) { @@ -137,8 +145,6 @@ struct SampleProfTest : ::testing::Test { ASSERT_TRUE(NoError(EC)); } - ASSERT_EQ(2u, Reader->getProfiles().size()); - FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName); ASSERT_TRUE(ReadFooSamples != nullptr); if (Format != SampleProfileFormat::SPF_Compact_Binary) { @@ -158,6 +164,20 @@ struct SampleProfTest : ::testing::Test { ReadBarSamples->findCallTargetMapAt(1, 0); ASSERT_FALSE(CTMap.getError()); + // Because _Z3bazi is not defined in module M, expect _Z3bazi's profile + // is not loaded when the profile is ExtBinary or Compact format because + // these formats support loading function profiles on demand. + FunctionSamples *ReadBazSamples = Reader->getSamplesFor(BazName); + if (Format == SampleProfileFormat::SPF_Ext_Binary || + Format == SampleProfileFormat::SPF_Compact_Binary) { + ASSERT_TRUE(ReadBazSamples == nullptr); + ASSERT_EQ(2u, Reader->getProfiles().size()); + } else { + ASSERT_TRUE(ReadBazSamples != nullptr); + ASSERT_EQ(12557u, ReadBazSamples->getTotalSamples()); + ASSERT_EQ(3u, Reader->getProfiles().size()); + } + std::string MconstructGUID; StringRef MconstructRep = getRepInFormat(MconstructName, Format, MconstructGUID); @@ -169,9 +189,9 @@ struct SampleProfTest : ::testing::Test { auto VerifySummary = [](ProfileSummary &Summary) mutable { ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind()); - ASSERT_EQ(123603u, Summary.getTotalCount()); - ASSERT_EQ(6u, Summary.getNumCounts()); - ASSERT_EQ(2u, Summary.getNumFunctions()); + ASSERT_EQ(136160u, Summary.getTotalCount()); + ASSERT_EQ(7u, Summary.getNumCounts()); + ASSERT_EQ(3u, Summary.getNumFunctions()); ASSERT_EQ(1437u, Summary.getMaxFunctionCount()); ASSERT_EQ(60351u, Summary.getMaxCount()); @@ -188,8 +208,8 @@ struct SampleProfTest : ::testing::Test { Cutoff = 990000; auto NinetyNinePerc = find_if(Details, Predicate); ASSERT_EQ(60000u, EightyPerc->MinCount); - ASSERT_EQ(60000u, NinetyPerc->MinCount); - ASSERT_EQ(60000u, NinetyFivePerc->MinCount); + ASSERT_EQ(12557u, NinetyPerc->MinCount); + ASSERT_EQ(12557u, NinetyFivePerc->MinCount); ASSERT_EQ(610u, NinetyNinePerc->MinCount); };