CodeGen: Fix implementation of __builtin_trivially_relocate.

.github/new-prs-labeler.yml

@@ -702,6 +702,7 @@ mlgo:
   - llvm/unittests/CodeGen/ML*
   - llvm/test/CodeGen/MLRegAlloc/**
   - llvm/utils/mlgo-utils/**
+  - llvm/docs/MLGO.rst
 
 tools:llvm-exegesis:
   - llvm/tools/llvm-exegesis/**

bolt/include/bolt/Profile/Heatmap.h

@@ -52,6 +52,9 @@ class Heatmap {
       : BucketSize(BucketSize), MinAddress(MinAddress), MaxAddress(MaxAddress),
         TextSections(TextSections) {}
 
+  uint64_t HotStart{0};
+  uint64_t HotEnd{0};
+
   inline bool ignoreAddress(uint64_t Address) const {
     return (Address > MaxAddress) || (Address < MinAddress);
   }

bolt/lib/Core/BinaryFunction.cpp

@@ -3326,7 +3326,7 @@ void BinaryFunction::duplicateConstantIslands() {
 static std::string constructFilename(std::string Filename,
                                      std::string Annotation,
                                      std::string Suffix) {
-  std::replace(Filename.begin(), Filename.end(), '/', '-');
+  llvm::replace(Filename, '/', '-');
   if (!Annotation.empty())
     Annotation.insert(0, "-");
   if (Filename.size() + Annotation.size() + Suffix.size() > MAX_PATH) {

bolt/lib/Core/DIEBuilder.cpp

@@ -437,10 +437,10 @@ getUnitForOffset(DIEBuilder &Builder, DWARFContext &DWCtx,
       // This is a work around for XCode clang. There is a build error when we
       // pass DWCtx.compile_units() to llvm::upper_bound
       std::call_once(InitVectorFlag, initCUVector);
-      auto CUIter = std::upper_bound(CUOffsets.begin(), CUOffsets.end(), Offset,
-                                     [](uint64_t LHS, const DWARFUnit *RHS) {
-                                       return LHS < RHS->getNextUnitOffset();
-                                     });
+      auto CUIter = llvm::upper_bound(CUOffsets, Offset,
+                                      [](uint64_t LHS, const DWARFUnit *RHS) {
+                                        return LHS < RHS->getNextUnitOffset();
+                                      });
       CU = CUIter != CUOffsets.end() ? (*CUIter) : nullptr;
     }
     return CU;

bolt/lib/Passes/AsmDump.cpp

@@ -109,7 +109,7 @@ void dumpFunction(const BinaryFunction &BF) {
   }
 
   std::string PrintName = BF.getPrintName();
-  std::replace(PrintName.begin(), PrintName.end(), '/', '-');
+  llvm::replace(PrintName, '/', '-');
   std::string Filename =
       opts::AsmDump.empty()
           ? (PrintName + ".s")

bolt/lib/Passes/BinaryPasses.cpp

@@ -35,7 +35,7 @@ static const char *dynoStatsOptName(const bolt::DynoStats::Category C) {
 
   OptNames[C] = bolt::DynoStats::Description(C);
 
-  std::replace(OptNames[C].begin(), OptNames[C].end(), ' ', '-');
+  llvm::replace(OptNames[C], ' ', '-');
 
   return OptNames[C].c_str();
 }

bolt/lib/Profile/DataAggregator.cpp

@@ -1316,6 +1316,14 @@ std::error_code DataAggregator::printLBRHeatMap() {
   }
   Heatmap HM(opts::HeatmapBlock, opts::HeatmapMinAddress,
              opts::HeatmapMaxAddress, getTextSections(BC));
+  auto getSymbolValue = [&](const MCSymbol *Symbol) -> uint64_t {
+    if (Symbol)
+      if (ErrorOr<uint64_t> SymValue = BC->getSymbolValue(*Symbol))
+        return SymValue.get();
+    return 0;
+  };
+  HM.HotStart = getSymbolValue(BC->getHotTextStartSymbol());
+  HM.HotEnd = getSymbolValue(BC->getHotTextEndSymbol());
 
   if (!NumTotalSamples) {
     if (opts::BasicAggregation) {

bolt/lib/Profile/Heatmap.cpp

@@ -8,6 +8,7 @@
 
 #include "bolt/Profile/Heatmap.h"
 #include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/ADT/AddressRanges.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
@@ -313,6 +314,9 @@ void Heatmap::printSectionHotness(raw_ostream &OS) const {
     UnmappedHotness += Frequency;
   };
 
+  AddressRange HotTextRange(HotStart, HotEnd);
+  StringRef HotTextName = "[hot text]";
+
   for (const std::pair<const uint64_t, uint64_t> &KV : Map) {
     NumTotalCounts += KV.second;
     // We map an address bucket to the first section (lowest address)
@@ -328,15 +332,24 @@ void Heatmap::printSectionHotness(raw_ostream &OS) const {
     }
     SectionHotness[TextSections[TextSectionIndex].Name] += KV.second;
     ++BucketUtilization[TextSections[TextSectionIndex].Name];
+    if (HotTextRange.contains(Address)) {
+      SectionHotness[HotTextName] += KV.second;
+      ++BucketUtilization[HotTextName];
+    }
   }
 
+  std::vector<SectionNameAndRange> Sections(TextSections);
+  // Append synthetic hot text section to TextSections
+  if (!HotTextRange.empty())
+    Sections.emplace_back(SectionNameAndRange{HotTextName, HotStart, HotEnd});
+
   assert(NumTotalCounts > 0 &&
          "total number of heatmap buckets should be greater than 0");
 
   OS << "Section Name, Begin Address, End Address, Percentage Hotness, "
      << "Utilization Pct, Partition Score\n";
   const uint64_t MappedCounts = NumTotalCounts - UnmappedHotness;
-  for (const auto [Name, Begin, End] : TextSections) {
+  for (const auto [Name, Begin, End] : Sections) {
     const float Hotness = 1. * SectionHotness[Name] / NumTotalCounts;
     const float MappedHotness =
         MappedCounts ? 1. * SectionHotness[Name] / MappedCounts : 0;

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -968,8 +968,9 @@ void RewriteInstance::discoverFileObjects() {
       continue;
     }
 
-    // Ignore input hot markers
-    if (SymName == "__hot_start" || SymName == "__hot_end")
+    // Ignore input hot markers unless in heatmap mode
+    if ((SymName == "__hot_start" || SymName == "__hot_end") &&
+        !opts::HeatmapMode)
       continue;
 
     FileSymRefs.emplace(SymbolAddress, Symbol);

bolt/test/X86/callcont-fallthru.s

@@ -6,7 +6,7 @@
 # RUN: %clangxx %cxxflags %s %t.so -o %t -Wl,-q -nostdlib
 # RUN: link_fdata %s %t %t.pat PREAGGT1
 # RUN: link_fdata %s %t %t.pat2 PREAGGT2
-# RUN: link_fdata %s %t %t.patplt PREAGGPLT
+# RUN-DISABLED: link_fdata %s %t %t.patplt PREAGGPLT
 
 # RUN: llvm-strip --strip-unneeded %t -o %t.strip
 # RUN: llvm-objcopy --remove-section=.eh_frame %t.strip %t.noeh
@@ -26,8 +26,8 @@
 
 ## Check pre-aggregated traces don't report zero-sized PLT fall-through as
 ## invalid trace
-# RUN: llvm-bolt %t.strip --pa -p %t.patplt -o %t.out | FileCheck %s \
-# RUN:   --check-prefix=CHECK-PLT
+# RUN-DISABLED: llvm-bolt %t.strip --pa -p %t.patplt -o %t.out | FileCheck %s \
+# RUN-DISABLED:   --check-prefix=CHECK-PLT
 # CHECK-PLT: traces mismatching disassembled function contents: 0
 
   .globl foo

bolt/test/X86/heatmap-preagg.test

@@ -13,6 +13,7 @@ RUN:   --reorder-functions=cdsort --enable-bat --dyno-stats --skip-funcs=main
 RUN: llvm-bolt-heatmap %t.out -o %t2 --pa -p %p/Inputs/blarge_new_bat.preagg.txt \
 RUN:   2>&1 | FileCheck --check-prefix CHECK-HEATMAP-BAT %s
 RUN: FileCheck %s --check-prefix CHECK-SEC-HOT-BAT --input-file %t2-section-hotness.csv
+RUN: llvm-nm -n %t.out | FileCheck %s --check-prefix=CHECK-HOT-SYMS
 
 CHECK-HEATMAP: PERF2BOLT: read 81 aggregated LBR entries
 CHECK-HEATMAP: HEATMAP: invalid traces: 1
@@ -33,3 +34,6 @@ CHECK-SEC-HOT-BAT-NEXT: .bolt.org.text, 0x4010b0, 0x401c25, 38.3385, 51.0638, 0.
 CHECK-SEC-HOT-BAT-NEXT: .fini, 0x401c28, 0x401c35, 0.0000, 0.0000, 0.0000
 CHECK-SEC-HOT-BAT-NEXT: .text, 0x800000, 0x8002cc, 38.7595, 91.6667, 0.3553
 CHECK-SEC-HOT-BAT-NEXT: .text.cold, 0x800300, 0x800415, 0.0000, 0.0000, 0.0000
+CHECK-SEC-HOT-BAT-NEXT: [hot text], 0x800000, 0x8002cc, 38.7595, 91.6667, 0.3553
+CHECK-HOT-SYMS: 800000 W __hot_start
+CHECK-HOT-SYMS: 8002cc W __hot_end

clang-tools-extra/clang-doc/HTMLGenerator.cpp

@@ -1022,12 +1022,12 @@ static llvm::Error serializeIndex(ClangDocContext &CDCtx) {
   // JavaScript from escaping characters incorrectly, and introducing  bad paths
   // in the URLs.
   std::string RootPathEscaped = RootPath.str().str();
-  std::replace(RootPathEscaped.begin(), RootPathEscaped.end(), '\\', '/');
+  llvm::replace(RootPathEscaped, '\\', '/');
   OS << "var RootPath = \"" << RootPathEscaped << "\";\n";
 
   llvm::SmallString<128> Base(CDCtx.Base);
   std::string BaseEscaped = Base.str().str();
-  std::replace(BaseEscaped.begin(), BaseEscaped.end(), '\\', '/');
+  llvm::replace(BaseEscaped, '\\', '/');
   OS << "var Base = \"" << BaseEscaped << "\";\n";
 
   CDCtx.Idx.sort();

clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp

@@ -49,9 +49,9 @@ std::string LLVMHeaderGuardCheck::getHeaderGuard(StringRef Filename,
   if (PosLLVM != StringRef::npos)
     Guard = Guard.substr(PosLLVM);
 
-  std::replace(Guard.begin(), Guard.end(), '/', '_');
-  std::replace(Guard.begin(), Guard.end(), '.', '_');
-  std::replace(Guard.begin(), Guard.end(), '-', '_');
+  llvm::replace(Guard, '/', '_');
+  llvm::replace(Guard, '.', '_');
+  llvm::replace(Guard, '-', '_');
 
   // The prevalent style in clang is LLVM_CLANG_FOO_BAR_H
   if (StringRef(Guard).starts_with("clang"))

clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp

@@ -496,7 +496,7 @@ void IdentifierNamingCheck::HungarianNotation::loadFileConfig(
     StringRef Val = Options.get(Buffer, "");
     if (!Val.empty()) {
       std::string Type = PrimType.str();
-      std::replace(Type.begin(), Type.end(), '-', ' ');
+      llvm::replace(Type, '-', ' ');
       HNOption.PrimitiveType[Type] = Val.str();
     }
   }
@@ -1358,7 +1358,7 @@ IdentifierNamingCheck::getFailureInfo(
   std::string KindName =
       fixupWithCase(Type, StyleNames[SK], ND, Style, HNOption,
                     IdentifierNamingCheck::CT_LowerCase);
-  std::replace(KindName.begin(), KindName.end(), '_', ' ');
+  llvm::replace(KindName, '_', ' ');
 
   std::string Fixup = fixupWithStyle(Type, Name, Style, HNOption, ND);
   if (StringRef(Fixup) == Name) {

clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp

@@ -202,8 +202,7 @@ bool MagicNumbersCheck::isIgnoredValue(const IntegerLiteral *Literal) const {
   if (IgnorePowersOf2IntegerValues && IntValue.isPowerOf2())
     return true;
 
-  return std::binary_search(IgnoredIntegerValues.begin(),
-                            IgnoredIntegerValues.end(), Value);
+  return llvm::binary_search(IgnoredIntegerValues, Value);
 }
 
 bool MagicNumbersCheck::isIgnoredValue(const FloatingLiteral *Literal) const {
@@ -213,14 +212,12 @@ bool MagicNumbersCheck::isIgnoredValue(const FloatingLiteral *Literal) const {
 
   if (&FloatValue.getSemantics() == &llvm::APFloat::IEEEsingle()) {
     const float Value = FloatValue.convertToFloat();
-    return std::binary_search(IgnoredFloatingPointValues.begin(),
-                              IgnoredFloatingPointValues.end(), Value);
+    return llvm::binary_search(IgnoredFloatingPointValues, Value);
   }
 
   if (&FloatValue.getSemantics() == &llvm::APFloat::IEEEdouble()) {
     const double Value = FloatValue.convertToDouble();
-    return std::binary_search(IgnoredDoublePointValues.begin(),
-                              IgnoredDoublePointValues.end(), Value);
+    return llvm::binary_search(IgnoredDoublePointValues, Value);
   }
 
   return false;

clang-tools-extra/clangd/Diagnostics.cpp

@@ -800,7 +800,7 @@ void StoreDiags::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
           M << "'";
         }
         // Don't allow source code to inject newlines into diagnostics.
-        std::replace(Message.begin(), Message.end(), '\n', ' ');
+        llvm::replace(Message, '\n', ' ');
       }
     }
     if (Message.empty()) // either !SyntheticMessage, or we failed to make one.

clang-tools-extra/clangd/FindTarget.cpp

@@ -57,7 +57,7 @@ LLVM_ATTRIBUTE_UNUSED std::string nodeToString(const DynTypedNode &N) {
     OS << ": ";
     N.print(OS, PrintingPolicy(LangOptions()));
   }
-  std::replace(S.begin(), S.end(), '\n', ' ');
+  llvm::replace(S, '\n', ' ');
   return S;
 }
 

clang-tools-extra/clangd/index/FileIndex.cpp

@@ -79,7 +79,8 @@ SlabTuple indexSymbols(ASTContext &AST, Preprocessor &PP,
 
   SymbolCollector Collector(std::move(CollectorOpts));
   Collector.setPreprocessor(PP);
-  index::indexTopLevelDecls(AST, PP, DeclsToIndex, Collector, IndexOpts);
+  index::indexTopLevelDecls(AST, PP, DeclsToIndex, Collector,
+                            std::move(IndexOpts));
   if (MacroRefsToIndex)
     Collector.handleMacros(*MacroRefsToIndex);
 

clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

@@ -381,7 +381,7 @@ std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
 
 bool runCommand(std::string Request, const SymbolIndex &Index) {
   // Split on spaces and add required null-termination.
-  std::replace(Request.begin(), Request.end(), ' ', '\0');
+  llvm::replace(Request, ' ', '\0');
   llvm::SmallVector<llvm::StringRef> Args;
   llvm::StringRef(Request).split(Args, '\0', /*MaxSplit=*/-1,
                                  /*KeepEmpty=*/false);

clang-tools-extra/modularize/ModularizeUtilities.cpp

@@ -443,7 +443,7 @@ static std::string replaceDotDot(StringRef Path) {
 // \returns The file path in canonical form.
 std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
   std::string Tmp(replaceDotDot(FilePath));
-  std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
+  llvm::replace(Tmp, '\\', '/');
   StringRef Tmp2(Tmp);
   if (Tmp2.starts_with("./"))
     Tmp = std::string(Tmp2.substr(2));

clang-tools-extra/modularize/ModuleAssistant.cpp

@@ -156,8 +156,8 @@ ensureNoCollisionWithReservedName(llvm::StringRef MightBeReservedName) {
 static std::string
 ensureVaidModuleName(llvm::StringRef MightBeInvalidName) {
   std::string SafeName(MightBeInvalidName);
-  std::replace(SafeName.begin(), SafeName.end(), '-', '_');
-  std::replace(SafeName.begin(), SafeName.end(), '.', '_');
+  llvm::replace(SafeName, '-', '_');
+  llvm::replace(SafeName, '.', '_');
   if (isdigit(SafeName[0]))
     SafeName = "_" + SafeName;
   return SafeName;
@@ -192,7 +192,7 @@ static bool addModuleDescription(Module *RootModule,
     return true;
   }
   // Make canonical.
-  std::replace(FilePath.begin(), FilePath.end(), '\\', '/');
+  llvm::replace(FilePath, '\\', '/');
   // Insert module into tree, using subdirectories as submodules.
   for (llvm::sys::path::const_iterator I = llvm::sys::path::begin(FilePath),
                                        E = llvm::sys::path::end(FilePath);

clang-tools-extra/modularize/PreprocessorTracker.cpp

@@ -904,7 +904,7 @@ class PreprocessorTrackerImpl : public PreprocessorTracker {
   // Convert to a canonical path.
   std::string getCanonicalPath(llvm::StringRef path) const {
     std::string CanonicalPath(path);
-    std::replace(CanonicalPath.begin(), CanonicalPath.end(), '\\', '/');
+    llvm::replace(CanonicalPath, '\\', '/');
     return CanonicalPath;
   }
 

clang-tools-extra/pp-trace/PPCallbacksTracker.cpp

@@ -44,7 +44,7 @@ static std::string getSourceLocationString(Preprocessor &PP,
     std::string Result = SS.str();
 
     // YAML treats backslash as escape, so use forward slashes.
-    std::replace(Result.begin(), Result.end(), '\\', '/');
+    llvm::replace(Result, '\\', '/');
 
     return Result;
   }
@@ -653,7 +653,7 @@ void PPCallbacksTracker::appendFilePathArgument(const char *Name,
                                                 llvm::StringRef Value) {
   std::string Path(Value);
   // YAML treats backslash as escape, so use forward slashes.
-  std::replace(Path.begin(), Path.end(), '\\', '/');
+  llvm::replace(Path, '\\', '/');
   appendQuotedArgument(Name, Path);
 }