InstrProf: Support for value profiling in the indexed profile format

Add support to the indexed instrprof reader and writer for the format
that will be used for value profiling.

Patch by Betul Buyukkurt, with minor modifications.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@248833 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/ProfileData/InstrProf.h

@@ -16,34 +16,95 @@
 #ifndef LLVM_PROFILEDATA_INSTRPROF_H_
 #define LLVM_PROFILEDATA_INSTRPROF_H_
 
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <cstdint>
+#include <list>
 #include <system_error>
 #include <vector>
 
 namespace llvm {
 const std::error_category &instrprof_category();
 
 enum class instrprof_error {
-    success = 0,
-    eof,
-    bad_magic,
-    bad_header,
-    unsupported_version,
-    unsupported_hash_type,
-    too_large,
-    truncated,
-    malformed,
-    unknown_function,
-    hash_mismatch,
-    count_mismatch,
-    counter_overflow
+  success = 0,
+  eof,
+  bad_magic,
+  bad_header,
+  unsupported_version,
+  unsupported_hash_type,
+  too_large,
+  truncated,
+  malformed,
+  unknown_function,
+  hash_mismatch,
+  count_mismatch,
+  counter_overflow,
+  value_site_count_mismatch
 };
 
 inline std::error_code make_error_code(instrprof_error E) {
   return std::error_code(static_cast<int>(E), instrprof_category());
 }
 
+enum InstrProfValueKind : uint32_t {
+  IPVK_IndirectCallTarget = 0,
+
+  IPVK_First = IPVK_IndirectCallTarget,
+  IPVK_Last = IPVK_IndirectCallTarget
+};
+
+struct InstrProfStringTable {
+  // Set of string values in profiling data.
+  StringSet<> StringValueSet;
+  InstrProfStringTable() { StringValueSet.clear(); }
+  // Get a pointer to internal storage of a string in set
+  const char *getStringData(StringRef Str) {
+    auto Result = StringValueSet.find(Str);
+    return (Result == StringValueSet.end()) ? nullptr : Result->first().data();
+  }
+  // Insert a string to StringTable
+  const char *insertString(StringRef Str) {
+    auto Result = StringValueSet.insert(Str);
+    return Result.first->first().data();
+  }
+};
+
+struct InstrProfValueSiteRecord {
+  /// Typedef for a single TargetValue-NumTaken pair.
+  typedef std::pair<uint64_t, uint64_t> ValueDataPair;
+  /// Value profiling data pairs at a given value site.
+  std::list<ValueDataPair> ValueData;
+
+  InstrProfValueSiteRecord() { ValueData.clear(); }
+
+  /// Sort ValueData ascending by TargetValue
+  void sortByTargetValues() {
+    ValueData.sort([](const ValueDataPair &left, const ValueDataPair &right) {
+      return left.first < right.first;
+    });
+  }
+
+  /// Merge data from another InstrProfValueSiteRecord
+  void mergeValueData(InstrProfValueSiteRecord &Input) {
+    this->sortByTargetValues();
+    Input.sortByTargetValues();
+    auto I = ValueData.begin();
+    auto IE = ValueData.end();
+    for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
+         ++J) {
+      while (I != IE && I->first < J->first)
+        ++I;
+      if (I != IE && I->first == J->first) {
+        I->second += J->second;
+        ++I;
+        continue;
+      }
+      ValueData.insert(I, *J);
+    }
+  }
+};
+
 /// Profiling information for a single function.
 struct InstrProfRecord {
   InstrProfRecord() {}
@@ -52,6 +113,22 @@ struct InstrProfRecord {
   StringRef Name;
   uint64_t Hash;
   std::vector<uint64_t> Counts;
+  std::vector<InstrProfValueSiteRecord> IndirectCallSites;
+
+  const std::vector<InstrProfValueSiteRecord> &
+  getValueSitesForKind(uint32_t ValueKind) const {
+    switch (ValueKind) {
+    case IPVK_IndirectCallTarget:
+      return IndirectCallSites;
+    }
+    llvm_unreachable("Unknown value kind!");
+  }
+
+  std::vector<InstrProfValueSiteRecord> &
+  getValueSitesForKind(uint32_t ValueKind) {
+    return const_cast<std::vector<InstrProfValueSiteRecord> &>(
+        this->getValueSitesForKind(ValueKind));
+  }
 };
 
 } // end namespace llvm

include/llvm/ProfileData/InstrProfReader.h

@@ -65,6 +65,9 @@ class InstrProfReader {
   InstrProfIterator end() { return InstrProfIterator(); }
 
 protected:
+  /// String table for holding a unique copy of all the strings in the profile.
+  InstrProfStringTable StringTable;
+
   /// Set the current std::error_code and return same.
   std::error_code error(std::error_code EC) {
     LastError = EC;
@@ -195,6 +198,7 @@ class InstrProfLookupTrait {
   std::vector<InstrProfRecord> DataBuffer;
   IndexedInstrProf::HashT HashType;
   unsigned FormatVersion;
+  std::vector<std::pair<uint64_t, const char *>> HashKeys;
 
 public:
   InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
@@ -209,9 +213,13 @@ class InstrProfLookupTrait {
 
   static bool EqualKey(StringRef A, StringRef B) { return A == B; }
   static StringRef GetInternalKey(StringRef K) { return K; }
+  static StringRef GetExternalKey(StringRef K) { return K; }
 
   hash_value_type ComputeHash(StringRef K);
 
+  void setHashKeys(std::vector<std::pair<uint64_t, const char *>> HashKeys) {
+    this->HashKeys = std::move(HashKeys);
+  }
   static std::pair<offset_type, offset_type>
   ReadKeyDataLength(const unsigned char *&D) {
     using namespace support;
@@ -224,6 +232,8 @@ class InstrProfLookupTrait {
     return StringRef((const char *)D, N);
   }
 
+  bool ReadValueProfilingData(const unsigned char *&D,
+                              const unsigned char *const End);
   data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
 };
 

include/llvm/ProfileData/InstrProfWriter.h

@@ -15,33 +15,32 @@
 #ifndef LLVM_PROFILEDATA_INSTRPROFWRITER_H
 #define LLVM_PROFILEDATA_INSTRPROFWRITER_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include <vector>
 
 namespace llvm {
 
 /// Writer for instrumentation based profile data.
 class InstrProfWriter {
 public:
-  typedef SmallDenseMap<uint64_t, std::vector<uint64_t>, 1> CounterData;
+  typedef SmallDenseMap<uint64_t, InstrProfRecord, 1> ProfilingData;
+
 private:
-  StringMap<CounterData> FunctionData;
+  InstrProfStringTable StringTable;
+  StringMap<ProfilingData> FunctionData;
   uint64_t MaxFunctionCount;
 public:
   InstrProfWriter() : MaxFunctionCount(0) {}
 
+  /// Update string entries in profile data with references to StringTable.
+  void updateStringTableReferences(InstrProfRecord &I);
   /// Add function counts for the given function. If there are already counts
   /// for this function and the hash and number of counts match, each counter is
   /// summed.
-  std::error_code addFunctionCounts(StringRef FunctionName,
-                                    uint64_t FunctionHash,
-                                    ArrayRef<uint64_t> Counters);
+  std::error_code addRecord(InstrProfRecord &&I);
   /// Write the profile to \c OS
   void write(raw_fd_ostream &OS);
   /// Write the profile, returning the raw data. For testing.

lib/ProfileData/InstrProf.cpp

@@ -50,6 +50,8 @@ class InstrProfErrorCategoryType : public std::error_category {
       return "Function count mismatch";
     case instrprof_error::counter_overflow:
       return "Counter overflow";
+    case instrprof_error::value_site_count_mismatch:
+      return "Function's value site counts mismatch";
     }
     llvm_unreachable("A value of instrprof_error has no message.");
   }

lib/ProfileData/InstrProfIndexed.h

@@ -47,7 +47,7 @@ static inline uint64_t ComputeHash(HashT Type, StringRef K) {
 }
 
 const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
-const uint64_t Version = 2;
+const uint64_t Version = 3;
 const HashT HashType = HashT::MD5;
 }
 

lib/ProfileData/InstrProfReader.cpp

@@ -302,42 +302,102 @@ InstrProfLookupTrait::ComputeHash(StringRef K) {
 typedef InstrProfLookupTrait::data_type data_type;
 typedef InstrProfLookupTrait::offset_type offset_type;
 
+bool InstrProfLookupTrait::ReadValueProfilingData(
+    const unsigned char *&D, const unsigned char *const End) {
+
+  using namespace support;
+  // Read number of value kinds with value sites.
+  if (D + sizeof(uint64_t) > End)
+    return false;
+  uint64_t ValueKindCount = endian::readNext<uint64_t, little, unaligned>(D);
+
+  for (uint32_t Kind = 0; Kind < ValueKindCount; ++Kind) {
+
+    // Read value kind and number of value sites for kind.
+    if (D + 2 * sizeof(uint64_t) > End)
+      return false;
+    uint64_t ValueKind = endian::readNext<uint64_t, little, unaligned>(D);
+    uint64_t ValueSiteCount = endian::readNext<uint64_t, little, unaligned>(D);
+
+    std::vector<InstrProfValueSiteRecord> &ValueSites =
+        DataBuffer.back().getValueSitesForKind(ValueKind);
+    ValueSites.reserve(ValueSiteCount);
+    for (uint64_t VSite = 0; VSite < ValueSiteCount; ++VSite) {
+      // Read number of value data pairs at value site.
+      if (D + sizeof(uint64_t) > End)
+        return false;
+      uint64_t ValueDataCount =
+          endian::readNext<uint64_t, little, unaligned>(D);
+
+      // Check if there are as many ValueDataPairs as ValueDataCount in memory.
+      if (D + (ValueDataCount << 1) * sizeof(uint64_t) > End)
+        return false;
+
+      InstrProfValueSiteRecord VSiteRecord;
+      for (uint64_t VCount = 0; VCount < ValueDataCount; ++VCount) {
+        uint64_t Value = endian::readNext<uint64_t, little, unaligned>(D);
+        uint64_t NumTaken = endian::readNext<uint64_t, little, unaligned>(D);
+        switch (ValueKind) {
+        case IPVK_IndirectCallTarget: {
+          auto Result =
+              std::lower_bound(HashKeys.begin(), HashKeys.end(), Value,
+                               [](const std::pair<uint64_t, const char *> &LHS,
+                                  uint64_t RHS) { return LHS.first < RHS; });
+          assert(Result != HashKeys.end() &&
+                 "Hash does not match any known keys\n");
+          Value = (uint64_t)Result->second;
+          break;
+        }
+        }
+        VSiteRecord.ValueData.push_back(std::make_pair(Value, NumTaken));
+      }
+      ValueSites.push_back(std::move(VSiteRecord));
+    }
+  }
+  return true;
+}
+
 data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
                                          offset_type N) {
-
   // Check if the data is corrupt. If so, don't try to read it.
   if (N % sizeof(uint64_t))
     return data_type();
 
   DataBuffer.clear();
-  uint64_t NumCounts;
-  uint64_t NumEntries = N / sizeof(uint64_t);
   std::vector<uint64_t> CounterBuffer;
-  for (uint64_t I = 0; I < NumEntries; I += NumCounts) {
-    using namespace support;
-    // The function hash comes first.
-    uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
 
-    if (++I >= NumEntries)
+  using namespace support;
+  const unsigned char *End = D + N;
+  while (D < End) {
+    // Read hash
+    if (D + sizeof(uint64_t) >= End)
       return data_type();
+    uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
 
-    // In v1, we have at least one count.
-    // Later, we have the number of counts.
-    NumCounts = (1 == FormatVersion)
-                    ? NumEntries - I
-                    : endian::readNext<uint64_t, little, unaligned>(D);
-    if (1 != FormatVersion)
-      ++I;
-
-    // If we have more counts than data, this is bogus.
-    if (I + NumCounts > NumEntries)
+    // Initialize number of counters for FormatVersion == 1
+    uint64_t CountsSize = N / sizeof(uint64_t) - 1;
+    // If format version is different then read number of counters
+    if (FormatVersion != 1) {
+      if (D + sizeof(uint64_t) > End)
+        return data_type();
+      CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
+    }
+    // Read counter values
+    if (D + CountsSize * sizeof(uint64_t) > End)
       return data_type();
 
     CounterBuffer.clear();
-    for (unsigned J = 0; J < NumCounts; ++J)
+    CounterBuffer.reserve(CountsSize);
+    for (uint64_t J = 0; J < CountsSize; ++J)
       CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
 
     DataBuffer.push_back(InstrProfRecord(K, Hash, std::move(CounterBuffer)));
+
+    // Read value profiling data
+    if (FormatVersion > 2 && !ReadValueProfilingData(D, End)) {
+      DataBuffer.clear();
+      return data_type();
+    }
   }
   return DataBuffer;
 }
@@ -384,6 +444,18 @@ std::error_code IndexedInstrProfReader::readHeader() {
   Index.reset(InstrProfReaderIndex::Create(
       Start + HashOffset, Cur, Start,
       InstrProfLookupTrait(HashType, FormatVersion)));
+
+  // Form the map of hash values to const char* keys in profiling data.
+  std::vector<std::pair<uint64_t, const char *>> HashKeys;
+  for (auto Key : Index->keys()) {
+    const char *KeyTableRef = StringTable.insertString(Key);
+    HashKeys.push_back(std::make_pair(ComputeHash(HashType, Key), KeyTableRef));
+  }
+  std::sort(HashKeys.begin(), HashKeys.end(), less_first());
+  std::unique(HashKeys.begin(), HashKeys.end(), less_first());
+  HashKeys.erase(std::unique(HashKeys.begin(), HashKeys.end()), HashKeys.end());
+  // Set the hash key map for the InstrLookupTrait
+  Index->getInfoObj().setHashKeys(std::move(HashKeys));
   // Set up our iterator for readNextRecord.
   RecordIterator = Index->data_begin();