diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 0431c182276ec..3520034fb1c94 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -16,6 +16,8 @@ namespace llvm { namespace memprof { +struct MemProfRecord; + // The versions of the indexed MemProf format enum IndexedVersion : uint64_t { // Version 0: This version didn't have a version field. @@ -392,6 +394,12 @@ struct IndexedMemProfRecord { const unsigned char *Buffer, IndexedVersion Version); + // Convert IndexedMemProfRecord to MemProfRecord. Callback is used to + // translate CallStackId to call stacks with frames inline. + MemProfRecord toMemProfRecord( + std::function(const CallStackId)> Callback) + const; + // Returns the GUID for the function name after canonicalization. For // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are // mapped to functions using this GUID. diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h index 89f49a20a6089..1f84fefad03e3 100644 --- a/llvm/include/llvm/ProfileData/MemProfReader.h +++ b/llvm/include/llvm/ProfileData/MemProfReader.h @@ -70,8 +70,20 @@ class MemProfReader { Callback = std::bind(&MemProfReader::idToFrame, this, std::placeholders::_1); + auto CallStackCallback = [&](CallStackId CSId) { + llvm::SmallVector CallStack; + auto Iter = CSIdToCallStack.find(CSId); + assert(Iter != CSIdToCallStack.end()); + for (FrameId Id : Iter->second) + CallStack.push_back(Callback(Id)); + return CallStack; + }; + const IndexedMemProfRecord &IndexedRecord = Iter->second; - GuidRecord = {Iter->first, MemProfRecord(IndexedRecord, Callback)}; + GuidRecord = { + Iter->first, + IndexedRecord.toMemProfRecord(CallStackCallback), + }; Iter++; return Error::success(); } @@ -84,9 +96,7 @@ class MemProfReader { // Initialize the MemProfReader with the frame mappings and profile contents. MemProfReader( llvm::DenseMap FrameIdMap, - llvm::MapVector ProfData) - : IdToFrame(std::move(FrameIdMap)), - FunctionProfileData(std::move(ProfData)) {} + llvm::MapVector ProfData); protected: // A helper method to extract the frame from the IdToFrame map. @@ -97,6 +107,8 @@ class MemProfReader { } // A mapping from FrameId (a hash of the contents) to the frame. llvm::DenseMap IdToFrame; + // A mapping from CallStackId to the call stack. + llvm::DenseMap> CSIdToCallStack; // A mapping from function GUID, hash of the canonical function symbol to the // memprof profile data for that function, i.e allocation and callsite info. llvm::MapVector FunctionProfileData; diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 97414505f1c13..1ca0a02d3cbde 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -224,6 +224,24 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema, llvm_unreachable("unsupported MemProf version"); } +MemProfRecord IndexedMemProfRecord::toMemProfRecord( + std::function(const CallStackId)> Callback) + const { + MemProfRecord Record; + + for (const memprof::IndexedAllocationInfo &IndexedAI : AllocSites) { + memprof::AllocationInfo AI; + AI.Info = IndexedAI.Info; + AI.CallStack = Callback(IndexedAI.CSId); + Record.AllocSites.push_back(AI); + } + + for (memprof::CallStackId CSId : CallSiteIds) + Record.CallSites.push_back(Callback(CSId)); + + return Record; +} + GlobalValue::GUID IndexedMemProfRecord::getGUID(const StringRef FunctionName) { // Canonicalize the function name to drop suffixes such as ".llvm.". Note // we do not drop any ".__uniq." suffixes, as getCanonicalFnName does not drop diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp index 580867a9083fd..91556f036c777 100644 --- a/llvm/lib/ProfileData/MemProfReader.cpp +++ b/llvm/lib/ProfileData/MemProfReader.cpp @@ -183,6 +183,28 @@ std::string getBuildIdString(const SegmentEntry &Entry) { } } // namespace +MemProfReader::MemProfReader( + llvm::DenseMap FrameIdMap, + llvm::MapVector ProfData) + : IdToFrame(std::move(FrameIdMap)), + FunctionProfileData(std::move(ProfData)) { + // Populate CSId in each IndexedAllocationInfo and IndexedMemProfRecord + // while storing CallStack in CSIdToCallStack. + for (auto &KV : FunctionProfileData) { + IndexedMemProfRecord &Record = KV.second; + for (auto &AS : Record.AllocSites) { + CallStackId CSId = hashCallStack(AS.CallStack); + AS.CSId = CSId; + CSIdToCallStack.insert({CSId, AS.CallStack}); + } + for (auto &CS : Record.CallSites) { + CallStackId CSId = hashCallStack(CS); + Record.CallSiteIds.push_back(CSId); + CSIdToCallStack.insert({CSId, CS}); + } + } +} + Expected> RawMemProfReader::create(const Twine &Path, const StringRef ProfiledBinary, bool KeepName) { @@ -445,6 +467,7 @@ Error RawMemProfReader::mapRawProfileToRecords() { } CallStackId CSId = hashCallStack(Callstack); + CSIdToCallStack.insert({CSId, Callstack}); // We attach the memprof record to each function bottom-up including the // first non-inline frame. @@ -467,7 +490,10 @@ Error RawMemProfReader::mapRawProfileToRecords() { auto Result = FunctionProfileData.insert({Id, IndexedMemProfRecord()}); IndexedMemProfRecord &Record = Result.first->second; for (LocationPtr Loc : Locs) { + CallStackId CSId = hashCallStack(*Loc); + CSIdToCallStack.insert({CSId, *Loc}); Record.CallSites.push_back(*Loc); + Record.CallSiteIds.push_back(CSId); } } diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index 9cf307472d656..ab9227e9df881 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -21,9 +21,11 @@ using ::llvm::DILineInfo; using ::llvm::DILineInfoSpecifier; using ::llvm::DILocal; using ::llvm::StringRef; +using ::llvm::memprof::CallStackId; using ::llvm::memprof::CallStackMap; using ::llvm::memprof::Frame; using ::llvm::memprof::FrameId; +using ::llvm::memprof::IndexedAllocationInfo; using ::llvm::memprof::IndexedMemProfRecord; using ::llvm::memprof::MemInfoBlock; using ::llvm::memprof::MemProfReader; @@ -36,6 +38,7 @@ using ::llvm::memprof::SegmentEntry; using ::llvm::object::SectionedAddress; using ::llvm::symbolize::SymbolizableModule; using ::testing::Return; +using ::testing::SizeIs; class MockSymbolizer : public SymbolizableModule { public: @@ -432,4 +435,86 @@ TEST(MemProf, BaseMemProfReader) { EXPECT_THAT(Records[0].AllocSites[0].CallStack[1], FrameContains("bar", 10U, 2U, false)); } + +TEST(MemProf, IndexedMemProfRecordToMemProfRecord) { + // Verify that MemProfRecord can be constructed from IndexedMemProfRecord with + // CallStackIds only. + + llvm::DenseMap FrameIdMap; + Frame F1(1, 0, 0, false); + Frame F2(2, 0, 0, false); + Frame F3(3, 0, 0, false); + Frame F4(4, 0, 0, false); + FrameIdMap.insert({F1.hash(), F1}); + FrameIdMap.insert({F2.hash(), F2}); + FrameIdMap.insert({F3.hash(), F3}); + FrameIdMap.insert({F4.hash(), F4}); + + llvm::DenseMap> CallStackIdMap; + llvm::SmallVector CS1 = {F1.hash(), F2.hash()}; + llvm::SmallVector CS2 = {F1.hash(), F3.hash()}; + llvm::SmallVector CS3 = {F2.hash(), F3.hash()}; + llvm::SmallVector CS4 = {F2.hash(), F4.hash()}; + CallStackIdMap.insert({llvm::memprof::hashCallStack(CS1), CS1}); + CallStackIdMap.insert({llvm::memprof::hashCallStack(CS2), CS2}); + CallStackIdMap.insert({llvm::memprof::hashCallStack(CS3), CS3}); + CallStackIdMap.insert({llvm::memprof::hashCallStack(CS4), CS4}); + + IndexedMemProfRecord IndexedRecord; + IndexedAllocationInfo AI; + AI.CSId = llvm::memprof::hashCallStack(CS1); + IndexedRecord.AllocSites.push_back(AI); + AI.CSId = llvm::memprof::hashCallStack(CS2); + IndexedRecord.AllocSites.push_back(AI); + IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS3)); + IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS4)); + + bool CSIdMissing = false; + bool FrameIdMissing = false; + + auto Callback = [&](CallStackId CSId) -> llvm::SmallVector { + llvm::SmallVector CallStack; + llvm::SmallVector FrameIds; + + auto Iter = CallStackIdMap.find(CSId); + if (Iter == CallStackIdMap.end()) + CSIdMissing = true; + else + FrameIds = Iter->second; + + for (FrameId Id : FrameIds) { + Frame F(0, 0, 0, false); + auto Iter = FrameIdMap.find(Id); + if (Iter == FrameIdMap.end()) + FrameIdMissing = true; + else + F = Iter->second; + CallStack.push_back(F); + } + + return CallStack; + }; + + MemProfRecord Record = IndexedRecord.toMemProfRecord(Callback); + + // Make sure that all lookups are successful. + ASSERT_FALSE(CSIdMissing); + ASSERT_FALSE(FrameIdMissing); + + // Verify the contents of Record. + ASSERT_THAT(Record.AllocSites, SizeIs(2)); + ASSERT_THAT(Record.AllocSites[0].CallStack, SizeIs(2)); + EXPECT_EQ(Record.AllocSites[0].CallStack[0].hash(), F1.hash()); + EXPECT_EQ(Record.AllocSites[0].CallStack[1].hash(), F2.hash()); + ASSERT_THAT(Record.AllocSites[1].CallStack, SizeIs(2)); + EXPECT_EQ(Record.AllocSites[1].CallStack[0].hash(), F1.hash()); + EXPECT_EQ(Record.AllocSites[1].CallStack[1].hash(), F3.hash()); + ASSERT_THAT(Record.CallSites, SizeIs(2)); + ASSERT_THAT(Record.CallSites[0], SizeIs(2)); + EXPECT_EQ(Record.CallSites[0][0].hash(), F2.hash()); + EXPECT_EQ(Record.CallSites[0][1].hash(), F3.hash()); + ASSERT_THAT(Record.CallSites[1], SizeIs(2)); + EXPECT_EQ(Record.CallSites[1][0].hash(), F2.hash()); + EXPECT_EQ(Record.CallSites[1][1].hash(), F4.hash()); +} } // namespace