Merge branch 'llvm:main' into GISel_rembyconst

Author: jyli0116

.ci/compute_projects.py

@@ -77,6 +77,7 @@
 DEPENDENT_RUNTIMES_TO_TEST = {
     "clang": {"compiler-rt"},
     "clang-tools-extra": {"libc"},
+    "libc": {"libc"},
     ".ci": {"compiler-rt", "libc"},
 }
 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {

.ci/compute_projects_test.py

@@ -187,7 +187,7 @@ def test_top_level_file(self):
         self.assertEqual(env_variables["runtimes_check_targets"], "")
         self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
-    def test_exclude_runtiems_in_projects(self):
+    def test_exclude_libcxx_in_projects(self):
         env_variables = compute_projects.get_env_variables(
             ["libcxx/CMakeLists.txt"], "Linux"
         )
@@ -197,6 +197,16 @@ def test_exclude_runtiems_in_projects(self):
         self.assertEqual(env_variables["runtimes_check_targets"], "")
         self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
+    def test_include_libc_in_runtimes(self):
+        env_variables = compute_projects.get_env_variables(
+            ["libc/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "clang;lld")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "libc")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
+
     def test_exclude_docs(self):
         env_variables = compute_projects.get_env_variables(
             ["llvm/docs/CIBestPractices.rst"], "Linux"

.github/copilot-instructions.md

@@ -0,0 +1,4 @@
+When performing a code review, pay close attention to code modifying a function's
+control flow. Could the change result in the corruption of performance profile
+data? Could the change result in invalid debug information, in particular for
+branches and calls?

.github/new-prs-labeler.yml

@@ -632,6 +632,10 @@ llvm:instcombine:
   - llvm/test/Transforms/InstCombine/**
   - llvm/test/Transforms/InstSimplify/**
 
+llvm:vectorcombine:
+  - llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+  - llvm/test/Transforms/VectorCombine/**
+
 clangd:
   - clang-tools-extra/clangd/**
 

.github/workflows/hlsl-test-all.yaml

@@ -43,13 +43,13 @@ jobs:
       - name: Checkout OffloadTest
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-test-suite
+          repository: llvm/offload-test-suite
           ref: main
           path: OffloadTest
       - name: Checkout Golden Images
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-golden-images
+          repository: llvm/offload-golden-images
           ref: main
           path: golden-images
       - name: Setup Windows

.github/workflows/libcxx-restart-preempted-jobs.yaml

@@ -32,8 +32,11 @@ jobs:
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
         with:
           script: |
-            const failure_regex = /Process completed with exit code 1./
-            const preemption_regex = /(The runner has received a shutdown signal)|(The operation was canceled)/
+            // The "The run was canceled by" message comes from a user manually canceling a workflow
+            // the "higher priority" message comes from github canceling a workflow because the user updated the change.
+            // And the "exit code 1" message indicates a genuine failure.
+            const failure_regex = /(Process completed with exit code 1.)|(Canceling since a higher priority waiting request)|(The run was canceled by)/
+            const preemption_regex = /(The runner has received a shutdown signal)/
 
             const wf_run = context.payload.workflow_run
             core.notice(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
@@ -167,7 +170,7 @@ jobs:
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
         with:
           script: |
-            const FAILURE_REGEX = /Process completed with exit code 1./
+            const FAILURE_REGEX = /(Process completed with exit code 1.)|(Canceling since a higher priority waiting request)|(The run was canceled by)/
             const PREEMPTION_REGEX = /(The runner has received a shutdown signal)|(The operation was canceled)/
 
             function log(msg) {

bolt/include/bolt/Core/BinarySection.h

@@ -523,11 +523,6 @@ inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) {
   return Array;
 }
 
-inline uint8_t *copyByteArray(StringRef Buffer) {
-  return copyByteArray(reinterpret_cast<const uint8_t *>(Buffer.data()),
-                       Buffer.size());
-}
-
 inline uint8_t *copyByteArray(ArrayRef<char> Buffer) {
   return copyByteArray(reinterpret_cast<const uint8_t *>(Buffer.data()),
                        Buffer.size());

bolt/include/bolt/Core/DIEBuilder.h

@@ -20,8 +20,8 @@
 #include "llvm/CodeGen/DIE.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Allocator.h"
 
 #include <list>

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1393,7 +1393,7 @@ class MCPlusBuilder {
       return getTargetSymbol(BinaryExpr->getLHS());
 
     auto *SymbolRefExpr = dyn_cast<const MCSymbolRefExpr>(Expr);
-    if (SymbolRefExpr && SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None)
+    if (SymbolRefExpr && SymbolRefExpr->getSpecifier() == 0)
       return &SymbolRefExpr->getSymbol();
 
     return nullptr;

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -202,6 +202,9 @@ class RewriteInstance {
   /// Map code sections generated by BOLT.
   void mapCodeSections(BOLTLinker::SectionMapper MapSection);
 
+  /// Map code without relocating sections.
+  void mapCodeSectionsInPlace(BOLTLinker::SectionMapper MapSection);
+
   /// Map the rest of allocatable sections.
   void mapAllocatableSections(BOLTLinker::SectionMapper MapSection);
 

bolt/lib/Core/BinaryFunction.cpp

@@ -3328,10 +3328,7 @@ void BinaryFunction::duplicateConstantIslands() {
 
         // Update instruction reference
         Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor(
-            Inst,
-            MCSymbolRefExpr::create(ColdSymbol, MCSymbolRefExpr::VK_None,
-                                    *BC.Ctx),
-            *BC.Ctx, 0));
+            Inst, MCSymbolRefExpr::create(ColdSymbol, *BC.Ctx), *BC.Ctx, 0));
         ++OpNum;
       }
     }

bolt/lib/Core/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   Demangle
   MC
   MCDisassembler

bolt/lib/Core/DIEBuilder.cpp

@@ -14,11 +14,11 @@
 #include "llvm/CodeGen/DIE.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"

bolt/lib/Core/DebugNames.cpp

@@ -8,8 +8,8 @@
 
 #include "bolt/Core/DebugNames.h"
 #include "bolt/Core/BinaryContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/LEB128.h"
 #include <cstdint>

bolt/lib/Passes/BinaryPasses.cpp

@@ -1765,27 +1765,26 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
 
   if (opts::ShowDensity) {
     double Density = 0.0;
-    // Sorted by the density in descending order.
-    llvm::stable_sort(FuncDensityList,
-                      [&](const std::pair<double, uint64_t> &A,
-                          const std::pair<double, uint64_t> &B) {
-                        if (A.first != B.first)
-                          return A.first > B.first;
-                        return A.second < B.second;
-                      });
+    llvm::sort(FuncDensityList);
 
     uint64_t AccumulatedSamples = 0;
-    uint32_t I = 0;
     assert(opts::ProfileDensityCutOffHot <= 1000000 &&
            "The cutoff value is greater than 1000000(100%)");
-    while (AccumulatedSamples <
-               TotalSampleCount *
-                   static_cast<float>(opts::ProfileDensityCutOffHot) /
-                   1000000 &&
-           I < FuncDensityList.size()) {
-      AccumulatedSamples += FuncDensityList[I].second;
-      Density = FuncDensityList[I].first;
-      I++;
+    // Subtract samples in zero-density functions (no fall-throughs) from
+    // TotalSampleCount (not used anywhere below).
+    for (const auto [CurDensity, CurSamples] : FuncDensityList) {
+      if (CurDensity != 0.0)
+        break;
+      TotalSampleCount -= CurSamples;
+    }
+    const uint64_t CutoffSampleCount =
+        1.f * TotalSampleCount * opts::ProfileDensityCutOffHot / 1000000;
+    // Process functions in decreasing density order
+    for (const auto [CurDensity, CurSamples] : llvm::reverse(FuncDensityList)) {
+      if (AccumulatedSamples >= CutoffSampleCount)
+        break;
+      AccumulatedSamples += CurSamples;
+      Density = CurDensity;
     }
     if (Density == 0.0) {
       BC.errs() << "BOLT-WARNING: the output profile is empty or the "

bolt/lib/Passes/Instrumentation.cpp

@@ -666,8 +666,7 @@ Error Instrumentation::runOnFunctions(BinaryContext &BC) {
       auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); };
       const auto LEA = std::find_if(
           std::next(llvm::find_if(reverse(BB), IsLEA)), BB.rend(), IsLEA);
-      LEA->getOperand(4).setExpr(
-          MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *BC.Ctx));
+      LEA->getOperand(4).setExpr(MCSymbolRefExpr::create(Target, *BC.Ctx));
     } else {
       BC.errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";
     }

bolt/lib/Passes/PAuthGadgetScanner.cpp

@@ -1319,6 +1319,90 @@ shouldReportReturnGadget(const BinaryContext &BC, const MCInstReference &Inst,
   return make_gadget_report(RetKind, Inst, *RetReg);
 }
 
+/// While BOLT already marks some of the branch instructions as tail calls,
+/// this function tries to detect less obvious cases, assuming false positives
+/// are acceptable as long as there are not too many of them.
+///
+/// It is possible that not all the instructions classified as tail calls by
+/// this function are safe to be considered as such for the purpose of code
+/// transformations performed by BOLT. The intention of this function is to
+/// spot some of actually missed tail calls (and likely a number of unrelated
+/// indirect branch instructions) as long as this doesn't increase the amount
+/// of false positive reports unacceptably.
+static bool shouldAnalyzeTailCallInst(const BinaryContext &BC,
+                                      const BinaryFunction &BF,
+                                      const MCInstReference &Inst) {
+  // Some BC.MIB->isXYZ(Inst) methods simply delegate to MCInstrDesc::isXYZ()
+  // (such as isBranch at the time of writing this comment), some don't (such
+  // as isCall). For that reason, call MCInstrDesc's methods explicitly when
+  // it is important.
+  const MCInstrDesc &Desc =
+      BC.MII->get(static_cast<const MCInst &>(Inst).getOpcode());
+  // Tail call should be a branch (but not necessarily an indirect one).
+  if (!Desc.isBranch())
+    return false;
+
+  // Always analyze the branches already marked as tail calls by BOLT.
+  if (BC.MIB->isTailCall(Inst))
+    return true;
+
+  // Try to also check the branches marked as "UNKNOWN CONTROL FLOW" - the
+  // below is a simplified condition from BinaryContext::printInstruction.
+  bool IsUnknownControlFlow =
+      BC.MIB->isIndirectBranch(Inst) && !BC.MIB->getJumpTable(Inst);
+
+  if (BF.hasCFG() && IsUnknownControlFlow)
+    return true;
+
+  return false;
+}
+
+static std::optional<PartialReport<MCPhysReg>>
+shouldReportUnsafeTailCall(const BinaryContext &BC, const BinaryFunction &BF,
+                           const MCInstReference &Inst, const SrcState &S) {
+  static const GadgetKind UntrustedLRKind(
+      "untrusted link register found before tail call");
+
+  if (!shouldAnalyzeTailCallInst(BC, BF, Inst))
+    return std::nullopt;
+
+  // Not only the set of registers returned by getTrustedLiveInRegs() can be
+  // seen as a reasonable target-independent _approximation_ of "the LR", these
+  // are *exactly* those registers used by SrcSafetyAnalysis to initialize the
+  // set of trusted registers on function entry.
+  // Thus, this function basically checks that the precondition expected to be
+  // imposed by a function call instruction (which is hardcoded into the target-
+  // specific getTrustedLiveInRegs() function) is also respected on tail calls.
+  SmallVector<MCPhysReg> RegsToCheck = BC.MIB->getTrustedLiveInRegs();
+  LLVM_DEBUG({
+    traceInst(BC, "Found tail call inst", Inst);
+    traceRegMask(BC, "Trusted regs", S.TrustedRegs);
+  });
+
+  // In musl on AArch64, the _start function sets LR to zero and calls the next
+  // stage initialization function at the end, something along these lines:
+  //
+  //   _start:
+  //     mov     x30, #0
+  //     ; ... other initialization ...
+  //     b       _start_c ; performs "exit" system call at some point
+  //
+  // As this would produce a false positive for every executable linked with
+  // such libc, ignore tail calls performed by ELF entry function.
+  if (BC.StartFunctionAddress &&
+      *BC.StartFunctionAddress == Inst.getFunction()->getAddress()) {
+    LLVM_DEBUG({ dbgs() << "  Skipping tail call in ELF entry function.\n"; });
+    return std::nullopt;
+  }
+
+  // Returns at most one report per instruction - this is probably OK...
+  for (auto Reg : RegsToCheck)
+    if (!S.TrustedRegs[Reg])
+      return make_gadget_report(UntrustedLRKind, Inst, Reg);
+
+  return std::nullopt;
+}
+
 static std::optional<PartialReport<MCPhysReg>>
 shouldReportCallGadget(const BinaryContext &BC, const MCInstReference &Inst,
                        const SrcState &S) {
@@ -1478,6 +1562,9 @@ void FunctionAnalysisContext::findUnsafeUses(
     if (PacRetGadgetsOnly)
       return;
 
+    if (auto Report = shouldReportUnsafeTailCall(BC, BF, Inst, S))
+      Reports.push_back(*Report);
+
     if (auto Report = shouldReportCallGadget(BC, Inst, S))
       Reports.push_back(*Report);
     if (auto Report = shouldReportSigningOracle(BC, Inst, S))

bolt/lib/Rewrite/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   Core
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   JITLink
   MC
   Object

bolt/lib/Rewrite/DWARFRewriter.cpp

@@ -24,10 +24,10 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCObjectWriter.h"