[MemProf] Add basic summary section support (#141805)

This patch adds support for a basic MemProf summary section, which is
built along with the indexed MemProf profile (e.g. when reading the raw
or YAML profiles), and serialized through the indexed profile just after
the header.

Currently only 6 fields are written, specifically the number of contexts
(total, cold, hot), and the max context size (cold, warm, hot).

To support forwards and backwards compatibility for added fields in the
indexed profile, the number of fields serialized first. The code is
written to support forwards compatibility (reading newer profiles with
additional summary fields), and comments indicate how to implement
backwards compatibility (reading older profiles with fewer summary
fields) as needed.

Support is added to print the summary as YAML comments when displaying
both the raw and indexed profiles via `llvm-profdata show`. Because they
are YAML comments, the YAML reader ignores these (the summary is always
recomputed when building the indexed profile as described above).

This necessitated moving some options and a couple of interfaces out of
Analysis/MemoryProfileInfo.cpp and into the new
ProfileData/MemProfSummary.cpp file, as we need to classify context
hotness earlier and also compute context ids to build the summary from
older indexed profiles.

Author: teresajohnson

llvm/include/llvm/Analysis/MemoryProfileInfo.h

@@ -21,11 +21,6 @@
 namespace llvm {
 namespace memprof {
 
-/// Return the allocation type for a given set of memory profile values.
-LLVM_ABI AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
-                                     uint64_t AllocCount,
-                                     uint64_t TotalLifetime);
-
 /// Build callstack metadata from the provided list of call stack ids. Returns
 /// the resulting metadata node.
 LLVM_ABI MDNode *buildCallstackMetadata(ArrayRef<uint64_t> CallStack,

llvm/include/llvm/ProfileData/IndexedMemProfData.h

@@ -24,6 +24,7 @@
 
 namespace llvm {
 namespace memprof {
+class MemProfSummary;
 struct IndexedMemProfData {
   // A map to hold memprof data per function. The lower 64 bits obtained from
   // the md5 hash of the function name is used to index into the map.
@@ -89,7 +90,7 @@ struct IndexedMemProfData {
 Error writeMemProf(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData);
-
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum);
 } // namespace llvm
 #endif

llvm/include/llvm/ProfileData/InstrProfReader.h

@@ -22,6 +22,7 @@
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/ProfileData/MemProfSummary.h"
 #include "llvm/ProfileData/MemProfYAML.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -690,6 +691,8 @@ class IndexedMemProfReader {
   /// The MemProf version.
   memprof::IndexedVersion Version =
       static_cast<memprof::IndexedVersion>(memprof::MinimumSupportedVersion);
+  /// MemProf summary (if available, version >= 4).
+  std::unique_ptr<memprof::MemProfSummary> MemProfSum;
   /// MemProf profile schema (if available).
   memprof::MemProfSchema Schema;
   /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName).
@@ -725,6 +728,8 @@ class IndexedMemProfReader {
 
   // Return the entire MemProf profile.
   memprof::AllMemProfData getAllMemProfData() const;
+
+  memprof::MemProfSummary *getSummary() const { return MemProfSum.get(); }
 };
 
 /// Reader for the indexed binary instrprof format.
@@ -887,6 +892,11 @@ class IndexedInstrProfReader : public InstrProfReader {
     }
   }
 
+  /// Return the MemProf summary. Will be null if unavailable (version < 4).
+  memprof::MemProfSummary *getMemProfSummary() const {
+    return MemProfReader.getSummary();
+  }
+
   Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override;
   Error printBinaryIds(raw_ostream &OS) override;
 };

llvm/include/llvm/ProfileData/InstrProfWriter.h

@@ -22,6 +22,7 @@
 #include "llvm/ProfileData/DataAccessProf.h"
 #include "llvm/ProfileData/IndexedMemProfData.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProfSummaryBuilder.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <memory>
@@ -84,6 +85,10 @@ class InstrProfWriter {
 
   std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData;
 
+  // MemProf summary builder to which records are added as MemProf data is added
+  // to the writer.
+  memprof::MemProfSummaryBuilder MemProfSumBuilder;
+
 public:
   // For memprof testing, random hotness can be assigned to the contexts if
   // MemprofGenerateRandomHotness is enabled. The random seed can be either

llvm/include/llvm/ProfileData/MemProfSummary.h

@@ -0,0 +1,70 @@
+//===- MemProfSummary.h - MemProf summary support ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains MemProf summary support and related interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_MEMPROFSUMMARY_H
+#define LLVM_PROFILEDATA_MEMPROFSUMMARY_H
+
+#include "llvm/IR/ModuleSummaryIndex.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProf.h"
+
+namespace llvm {
+namespace memprof {
+
+/// Return the allocation type for a given set of memory profile values.
+AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
+                            uint64_t AllocCount, uint64_t TotalLifetime);
+
+/// Helper to generate a single hash id for a given callstack, used for emitting
+/// matching statistics and useful for uniquing such statistics across modules.
+/// Also used to dedup contexts when computing the summary.
+uint64_t computeFullStackId(ArrayRef<Frame> CallStack);
+
+class MemProfSummary {
+private:
+  /// The number of summary fields below, which is used to enable some forwards
+  /// and backwards compatibility for the summary when serialized in the indexed
+  /// MemProf format. As long as no existing summary fields are removed or
+  /// reordered, and new summary fields are added after existing summary fields,
+  /// the MemProf indexed profile version does not need to be bumped to
+  /// accommodate new summary fields.
+  static constexpr unsigned NumSummaryFields = 6;
+
+  const uint64_t NumContexts, NumColdContexts, NumHotContexts;
+  const uint64_t MaxColdTotalSize, MaxWarmTotalSize, MaxHotTotalSize;
+
+public:
+  MemProfSummary(uint64_t NumContexts, uint64_t NumColdContexts,
+                 uint64_t NumHotContexts, uint64_t MaxColdTotalSize,
+                 uint64_t MaxWarmTotalSize, uint64_t MaxHotTotalSize)
+      : NumContexts(NumContexts), NumColdContexts(NumColdContexts),
+        NumHotContexts(NumHotContexts), MaxColdTotalSize(MaxColdTotalSize),
+        MaxWarmTotalSize(MaxWarmTotalSize), MaxHotTotalSize(MaxHotTotalSize) {}
+
+  static constexpr unsigned getNumSummaryFields() { return NumSummaryFields; }
+  uint64_t getNumContexts() const { return NumContexts; }
+  uint64_t getNumColdContexts() const { return NumColdContexts; }
+  uint64_t getNumHotContexts() const { return NumHotContexts; }
+  uint64_t getMaxColdTotalSize() const { return MaxColdTotalSize; }
+  uint64_t getMaxWarmTotalSize() const { return MaxWarmTotalSize; }
+  uint64_t getMaxHotTotalSize() const { return MaxHotTotalSize; }
+  void printSummaryYaml(raw_ostream &OS) const;
+  /// Write to indexed MemProf profile.
+  void write(ProfOStream &OS) const;
+  /// Read from indexed MemProf profile.
+  static std::unique_ptr<MemProfSummary> deserialize(const unsigned char *&);
+};
+
+} // namespace memprof
+} // namespace llvm
+
+#endif // LLVM_PROFILEDATA_MEMPROFSUMMARY_H

llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h

@@ -0,0 +1,51 @@
+//===- MemProfSummaryBuilder.h - MemProf summary building -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains MemProf summary builder.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H
+#define LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H
+
+#include "llvm/ProfileData/MemProf.h"
+#include "llvm/ProfileData/MemProfSummary.h"
+
+namespace llvm {
+namespace memprof {
+
+class MemProfSummaryBuilder {
+private:
+  // The set of full context IDs that we've recorded so far. This is needed to
+  // dedup the MIBs, which are duplicated between functions containing inline
+  // instances of the same allocations.
+  DenseSet<uint64_t> Contexts;
+
+  void addRecord(uint64_t, const PortableMemInfoBlock &);
+
+protected:
+  uint64_t MaxColdTotalSize = 0;
+  uint64_t MaxWarmTotalSize = 0;
+  uint64_t MaxHotTotalSize = 0;
+  uint64_t NumContexts = 0;
+  uint64_t NumColdContexts = 0;
+  uint64_t NumHotContexts = 0;
+
+public:
+  MemProfSummaryBuilder() = default;
+  ~MemProfSummaryBuilder() = default;
+
+  void addRecord(const IndexedMemProfRecord &);
+  void addRecord(const MemProfRecord &);
+  std::unique_ptr<MemProfSummary> getSummary();
+};
+
+} // namespace memprof
+} // namespace llvm
+
+#endif // LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H

llvm/lib/Analysis/MemoryProfileInfo.cpp

@@ -21,34 +21,6 @@ using namespace llvm::memprof;
 
 #define DEBUG_TYPE "memory-profile-info"
 
-// Upper bound on lifetime access density (accesses per byte per lifetime sec)
-// for marking an allocation cold.
-LLVM_ABI cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
-    "memprof-lifetime-access-density-cold-threshold", cl::init(0.05),
-    cl::Hidden,
-    cl::desc("The threshold the lifetime access density (accesses per byte per "
-             "lifetime sec) must be under to consider an allocation cold"));
-
-// Lower bound on lifetime to mark an allocation cold (in addition to accesses
-// per byte per sec above). This is to avoid pessimizing short lived objects.
-LLVM_ABI cl::opt<unsigned> MemProfAveLifetimeColdThreshold(
-    "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden,
-    cl::desc("The average lifetime (s) for an allocation to be considered "
-             "cold"));
-
-// Lower bound on average lifetime accesses density (total life time access
-// density / alloc count) for marking an allocation hot.
-LLVM_ABI cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold(
-    "memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000),
-    cl::Hidden,
-    cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an "
-             "allocation to be considered hot"));
-
-LLVM_ABI cl::opt<bool>
-    MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden,
-                       cl::desc("Enable use of hot hints (only supported for "
-                                "unambigously hot allocations)"));
-
 cl::opt<bool> MemProfReportHintedSizes(
     "memprof-report-hinted-sizes", cl::init(false), cl::Hidden,
     cl::desc("Report total allocation sizes of hinted allocations"));
@@ -73,28 +45,6 @@ cl::opt<unsigned> MinCallsiteColdBytePercent(
     cl::desc("Min percent of cold bytes at a callsite to discard non-cold "
              "contexts"));
 
-AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity,
-                                           uint64_t AllocCount,
-                                           uint64_t TotalLifetime) {
-  // The access densities are multiplied by 100 to hold 2 decimal places of
-  // precision, so need to divide by 100.
-  if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
-          MemProfLifetimeAccessDensityColdThreshold
-      // Lifetime is expected to be in ms, so convert the threshold to ms.
-      && ((float)TotalLifetime) / AllocCount >=
-             MemProfAveLifetimeColdThreshold * 1000)
-    return AllocationType::Cold;
-
-  // The access densities are multiplied by 100 to hold 2 decimal places of
-  // precision, so need to divide by 100.
-  if (MemProfUseHotHints &&
-      ((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
-          MemProfMinAveLifetimeAccessDensityHotThreshold)
-    return AllocationType::Hot;
-
-  return AllocationType::NotCold;
-}
-
 MDNode *llvm::memprof::buildCallstackMetadata(ArrayRef<uint64_t> CallStack,
                                               LLVMContext &Ctx) {
   SmallVector<Metadata *, 8> StackVals;

llvm/lib/ProfileData/CMakeLists.txt

@@ -10,6 +10,8 @@ add_llvm_component_library(LLVMProfileData
   MemProf.cpp
   MemProfReader.cpp
   MemProfRadixTree.cpp
+  MemProfSummary.cpp
+  MemProfSummaryBuilder.cpp
   PGOCtxProfReader.cpp
   PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp

llvm/lib/ProfileData/IndexedMemProfData.cpp

@@ -15,6 +15,7 @@
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ProfileData/MemProfRadixTree.h"
+#include "llvm/ProfileData/MemProfSummary.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/OnDiskHashTable.h"
 
@@ -220,7 +221,8 @@ static Error writeMemProfRadixTreeBased(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion Version, bool MemProfFullSchema,
     std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData =
-        nullptr) {
+        nullptr,
+    memprof::MemProfSummary *MemProfSum = nullptr) {
   assert((Version == memprof::Version3 || Version == memprof::Version4) &&
          "Unsupported version for radix tree format");
 
@@ -229,9 +231,12 @@ static Error writeMemProfRadixTreeBased(
   OS.write(0ULL); // Reserve space for the memprof call stack payload offset.
   OS.write(0ULL); // Reserve space for the memprof record payload offset.
   OS.write(0ULL); // Reserve space for the memprof record table offset.
-  if (Version >= memprof::Version4)
+  if (Version >= memprof::Version4) {
     OS.write(0ULL); // Reserve space for the data access profile offset.
 
+    MemProfSum->write(OS);
+  }
+
   auto Schema = memprof::getHotColdSchema();
   if (MemProfFullSchema)
     Schema = memprof::getFullSchema();
@@ -297,25 +302,27 @@ static Error writeMemProfV3(ProfOStream &OS,
 static Error writeMemProfV4(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData) {
-  return writeMemProfRadixTreeBased(OS, MemProfData, memprof::Version4,
-                                    MemProfFullSchema,
-                                    std::move(DataAccessProfileData));
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum) {
+  return writeMemProfRadixTreeBased(
+      OS, MemProfData, memprof::Version4, MemProfFullSchema,
+      std::move(DataAccessProfileData), MemProfSum);
 }
 
 // Write out the MemProf data in a requested version.
 Error writeMemProf(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData) {
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum) {
   switch (MemProfVersionRequested) {
   case memprof::Version2:
     return writeMemProfV2(OS, MemProfData, MemProfFullSchema);
   case memprof::Version3:
     return writeMemProfV3(OS, MemProfData, MemProfFullSchema);
   case memprof::Version4:
     return writeMemProfV4(OS, MemProfData, MemProfFullSchema,
-                          std::move(DataAccessProfileData));
+                          std::move(DataAccessProfileData), MemProfSum);
   }
 
   return make_error<InstrProfError>(
@@ -395,9 +402,11 @@ Error IndexedMemProfReader::deserializeRadixTreeBased(
       support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
 
   uint64_t DataAccessProfOffset = 0;
-  if (Version == memprof::Version4)
+  if (Version >= memprof::Version4) {
     DataAccessProfOffset =
         support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+    MemProfSum = memprof::MemProfSummary::deserialize(Ptr);
+  }
 
   // Read the schema.
   auto SchemaOr = memprof::readMemProfSchema(Ptr);