[dsymutil] Add support for generating DWARF5 accelerator tables.

This patch add support for emitting DWARF5 accelerator tables
(.debug_names) from dsymutil. Just as with the Apple style accelerator
tables, it's possible to update existing dSYMs. This patch includes a
test that show how you can convert back and forth between the two types.

If no kind of table is specified, dsymutil will default to generating
Apple-style accelerator tables whenever it finds those in its input. The
same is true when there are no accelerator tables at all. Finally, in
the remaining case, where there's at least one DWARF v5 table and no
Apple ones, the output will contains a DWARF accelerator tables
(.debug_names).

Differential revision: https://reviews.llvm.org/D49137

llvm-svn: 337980

llvm/test/tools/dsymutil/X86/accelerator.test

@@ -0,0 +1,38 @@
+RUN: dsymutil -accelerator=Dwarf -oso-prepend-path=%p/.. %p/../Inputs/basic.macho.x86_64 -o %t.dwarf.dSYM
+RUN: dsymutil -accelerator=Apple -oso-prepend-path=%p/.. %p/../Inputs/basic.macho.x86_64 -o %t.apple.dSYM
+
+RUN: llvm-dwarfdump -verify %t.dwarf.dSYM
+RUN: llvm-dwarfdump -verify %t.apple.dSYM
+
+RUN: llvm-dwarfdump -debug-names %t.dwarf.dSYM | FileCheck %s -check-prefix=NAMES -check-prefix=DWARF
+RUN: llvm-dwarfdump -apple-names -apple-namespaces -apple-types %t.apple.dSYM | FileCheck %s -check-prefix=NAMES -check-prefix=APPLE
+
+RUN: dsymutil -update -accelerator=Dwarf %t.apple.dSYM -o %t.dwarf.updated.dSYM
+RUN: dsymutil -update -accelerator=Apple %t.dwarf.dSYM -o %t.apple.updated.dSYM
+
+RUN: llvm-dwarfdump -verify %t.dwarf.updated.dSYM
+RUN: llvm-dwarfdump -verify %t.apple.updated.dSYM
+
+RUN: llvm-dwarfdump -debug-names %t.dwarf.updated.dSYM | FileCheck %s -check-prefix=NAMES -check-prefix=DWARF
+RUN: llvm-dwarfdump -apple-names -apple-namespaces -apple-types %t.apple.updated.dSYM | FileCheck %s -check-prefix=NAMES -check-prefix=APPLE
+
+DWARF: .debug_names contents:
+DWARF: Compilation Unit offsets [
+DWARF:   CU[0]
+DWARF:   CU[1]
+DWARF:   CU[2]
+DWARF: ]
+
+APPLE-DAG: .apple_names contents:
+APPLE-DAG: .apple_types contents:
+APPLE-DAG: .apple_namespaces contents:
+
+NAMES-DAG: "private_int"
+NAMES-DAG: "baz"
+NAMES-DAG: "int"
+NAMES-DAG: "bar"
+NAMES-DAG: "foo"
+NAMES-DAG: "inc"
+NAMES-DAG: "val"
+NAMES-DAG: "main"
+NAMES-DAG: "char"

llvm/test/tools/dsymutil/cmdline.test

@@ -5,6 +5,7 @@ HELP-NOT: -reverse-iterate
 HELP: Color Options
 HELP: -color
 HELP: Specific Options:
+HELP: -accelerator
 HELP: -arch=<arch>
 HELP: -dump-debug-map
 HELP: -flat

llvm/tools/dsymutil/DwarfLinker.cpp

@@ -321,6 +321,7 @@ void DwarfLinker::startDebugObject(LinkContext &Context) {
 
 void DwarfLinker::endDebugObject(LinkContext &Context) {
   Context.Clear();
+
   for (auto I = DIEBlocks.begin(), E = DIEBlocks.end(); I != E; ++I)
     (*I)->~DIEBlock();
   for (auto I = DIELocs.begin(), E = DIELocs.end(); I != E; ++I)
@@ -1746,6 +1747,20 @@ void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit,
 }
 
 void DwarfLinker::emitAcceleratorEntriesForUnit(CompileUnit &Unit) {
+  switch (Options.TheAccelTableKind) {
+  case AccelTableKind::Apple:
+    emitAppleAcceleratorEntriesForUnit(Unit);
+    break;
+  case AccelTableKind::Dwarf:
+    emitDwarfAcceleratorEntriesForUnit(Unit);
+    break;
+  case AccelTableKind::Default:
+    llvm_unreachable("The default must be updated to a concrete value.");
+    break;
+  }
+}
+
+void DwarfLinker::emitAppleAcceleratorEntriesForUnit(CompileUnit &Unit) {
   // Add namespaces.
   for (const auto &Namespace : Unit.getNamespaces())
     AppleNamespaces.addName(Namespace.Name,
@@ -1774,6 +1789,18 @@ void DwarfLinker::emitAcceleratorEntriesForUnit(CompileUnit &Unit) {
     AppleObjc.addName(ObjC.Name, ObjC.Die->getOffset() + Unit.getStartOffset());
 }
 
+void DwarfLinker::emitDwarfAcceleratorEntriesForUnit(CompileUnit &Unit) {
+  for (const auto &Namespace : Unit.getNamespaces())
+    DebugNames.addName(Namespace.Name, Namespace.Die->getOffset(),
+                       Namespace.Die->getTag(), Unit.getUniqueID());
+  for (const auto &Pubname : Unit.getPubnames())
+    DebugNames.addName(Pubname.Name, Pubname.Die->getOffset(),
+                       Pubname.Die->getTag(), Unit.getUniqueID());
+  for (const auto &Pubtype : Unit.getPubtypes())
+    DebugNames.addName(Pubtype.Name, Pubtype.Die->getOffset(),
+                       Pubtype.Die->getTag(), Unit.getUniqueID());
+}
+
 /// Read the frame info stored in the object, and emit the
 /// patched frame descriptions for the linked binary.
 ///
@@ -2063,9 +2090,9 @@ Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
   // Setup access to the debug info.
   auto DwarfContext = DWARFContext::create(*ErrOrObj);
   RelocationManager RelocMgr(*this);
-  for (const auto &CU : DwarfContext->compile_units()) {
-    maybeUpdateMaxDwarfVersion(CU->getVersion());
 
+  for (const auto &CU : DwarfContext->compile_units()) {
+    updateDwarfVersion(CU->getVersion());
     // Recursively get all modules imported by this one.
     auto CUDie = CU->getUnitDIE(false);
     if (!CUDie)
@@ -2172,6 +2199,26 @@ void DwarfLinker::DIECloner::cloneAllCompileUnits(
   }
 }
 
+void DwarfLinker::updateAccelKind(DWARFContext &Dwarf) {
+  if (Options.TheAccelTableKind != AccelTableKind::Default)
+    return;
+
+  auto &DwarfObj = Dwarf.getDWARFObj();
+
+  if (!AtLeastOneDwarfAccelTable &&
+      (!DwarfObj.getAppleNamesSection().Data.empty() ||
+       !DwarfObj.getAppleTypesSection().Data.empty() ||
+       !DwarfObj.getAppleNamespacesSection().Data.empty() ||
+       !DwarfObj.getAppleObjCSection().Data.empty())) {
+    AtLeastOneAppleAccelTable = true;
+  }
+
+  if (!AtLeastOneDwarfAccelTable &&
+      !DwarfObj.getDebugNamesSection().Data.empty()) {
+    AtLeastOneDwarfAccelTable = true;
+  }
+}
+
 bool DwarfLinker::emitPaperTrailWarnings(const DebugMapObject &DMO,
                                          const DebugMap &Map,
                                          OffsetsStringPool &StringPool) {
@@ -2245,8 +2292,12 @@ bool DwarfLinker::link(const DebugMap &Map) {
   unsigned NumObjects = Map.getNumberOfObjects();
   std::vector<LinkContext> ObjectContexts;
   ObjectContexts.reserve(NumObjects);
-  for (const auto &Obj : Map.objects())
+  for (const auto &Obj : Map.objects()) {
     ObjectContexts.emplace_back(Map, *this, *Obj.get());
+    LinkContext &LC = ObjectContexts.back();
+    if (LC.ObjectFile)
+      updateAccelKind(*LC.DwarfContext);
+  }
 
   // This Dwarf string pool which is only used for uniquing. This one should
   // never be used for offsets as its not thread-safe or predictable.
@@ -2260,6 +2311,19 @@ bool DwarfLinker::link(const DebugMap &Map) {
   // ODR Contexts for the link.
   DeclContextTree ODRContexts;
 
+  // If we haven't decided on an accelerator table kind yet, we base ourselves
+  // on the DWARF we have seen so far. At this point we haven't pulled in debug
+  // information from modules yet, so it is technically possible that they
+  // would affect the decision. However, as they're built with the same
+  // compiler and flags, it is safe to assume that they will follow the
+  // decision made here.
+  if (Options.TheAccelTableKind == AccelTableKind::Default) {
+    if (AtLeastOneDwarfAccelTable && !AtLeastOneAppleAccelTable)
+      Options.TheAccelTableKind = AccelTableKind::Dwarf;
+    else
+      Options.TheAccelTableKind = AccelTableKind::Apple;
+  }
+
   for (LinkContext &LinkContext : ObjectContexts) {
     if (Options.Verbose)
       outs() << "DEBUG MAP OBJECT: " << LinkContext.DMO.getObjectFilename()
@@ -2325,7 +2389,9 @@ bool DwarfLinker::link(const DebugMap &Map) {
     // In a first phase, just read in the debug info and load all clang modules.
     LinkContext.CompileUnits.reserve(
         LinkContext.DwarfContext->getNumCompileUnits());
+
     for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
+      updateDwarfVersion(CU->getVersion());
       auto CUDie = CU->getUnitDIE(false);
       if (Options.Verbose) {
         outs() << "Input compilation unit:";
@@ -2341,13 +2407,11 @@ bool DwarfLinker::link(const DebugMap &Map) {
                                    UniquingStringPool, ODRContexts, UnitID)) {
         LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
             *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
-        maybeUpdateMaxDwarfVersion(CU->getVersion());
       }
     }
   }
 
-  // If we haven't seen any CUs, pick an arbitrary valid Dwarf version anyway,
-  // to be able to emit papertrail warnings.
+  // If we haven't seen any CUs, pick an arbitrary valid Dwarf version anyway.
   if (MaxDwarfVersion == 0)
     MaxDwarfVersion = 3;
 
@@ -2444,10 +2508,20 @@ bool DwarfLinker::link(const DebugMap &Map) {
     if (!Options.NoOutput) {
       Streamer->emitAbbrevs(Abbreviations, MaxDwarfVersion);
       Streamer->emitStrings(OffsetsStringPool);
-      Streamer->emitAppleNames(AppleNames);
-      Streamer->emitAppleNamespaces(AppleNamespaces);
-      Streamer->emitAppleTypes(AppleTypes);
-      Streamer->emitAppleObjc(AppleObjc);
+      switch (Options.TheAccelTableKind) {
+      case AccelTableKind::Apple:
+        Streamer->emitAppleNames(AppleNames);
+        Streamer->emitAppleNamespaces(AppleNamespaces);
+        Streamer->emitAppleTypes(AppleTypes);
+        Streamer->emitAppleObjc(AppleObjc);
+        break;
+      case AccelTableKind::Dwarf:
+        Streamer->emitDebugNames(DebugNames);
+        break;
+      case AccelTableKind::Default:
+        llvm_unreachable("Default should have already been resolved.");
+        break;
+      }
     }
   };
 
@@ -2465,7 +2539,7 @@ bool DwarfLinker::link(const DebugMap &Map) {
   }
 
   return Options.NoOutput ? true : Streamer->finish(Map);
-}
+} // namespace dsymutil
 
 bool linkDwarf(raw_fd_ostream &OutFile, BinaryHolder &BinHolder,
                const DebugMap &DM, const LinkOptions &Options) {

llvm/tools/dsymutil/DwarfLinker.h

@@ -67,12 +67,15 @@ class DwarfLinker {
                      const DWARFDie *DIE = nullptr) const;
 
 private:
-  /// Remembers the newest DWARF version we've seen in a unit.
-  void maybeUpdateMaxDwarfVersion(unsigned Version) {
-    if (MaxDwarfVersion < Version)
-      MaxDwarfVersion = Version;
+  /// Remembers the oldest and newest DWARF version we've seen in a unit.
+  void updateDwarfVersion(unsigned Version) {
+    MaxDwarfVersion = std::max(MaxDwarfVersion, Version);
+    MinDwarfVersion = std::min(MinDwarfVersion, Version);
   }
 
+  /// Remembers the kinds of accelerator tables we've seen in a unit.
+  void updateAccelKind(DWARFContext &Dwarf);
+
   /// Emit warnings as Dwarf compile units to leave a trail after linking.
   bool emitPaperTrailWarnings(const DebugMapObject &DMO, const DebugMap &Map,
                               OffsetsStringPool &StringPool);
@@ -158,8 +161,10 @@ class DwarfLinker {
       DwarfContext = ObjectFile ? DWARFContext::create(*ObjectFile) : nullptr;
     }
 
-    /// Clear compile units and ranges.
+    /// Clear part of the context that's no longer needed when we're done with
+    /// the debug object.
     void Clear() {
+      DwarfContext.reset(nullptr);
       CompileUnits.clear();
       Ranges.clear();
     }
@@ -411,6 +416,8 @@ class DwarfLinker {
 
   /// Emit the accelerator entries for \p Unit.
   void emitAcceleratorEntriesForUnit(CompileUnit &Unit);
+  void emitDwarfAcceleratorEntriesForUnit(CompileUnit &Unit);
+  void emitAppleAcceleratorEntriesForUnit(CompileUnit &Unit);
 
   /// Patch the frame info for an object file and emit it.
   void patchFrameInfoForObject(const DebugMapObject &, RangesTy &Ranges,
@@ -449,7 +456,12 @@ class DwarfLinker {
   LinkOptions Options;
   std::unique_ptr<DwarfStreamer> Streamer;
   uint64_t OutputDebugInfoSize;
+
   unsigned MaxDwarfVersion = 0;
+  unsigned MinDwarfVersion = std::numeric_limits<unsigned>::max();
+
+  bool AtLeastOneAppleAccelTable = false;
+  bool AtLeastOneDwarfAccelTable = false;
 
   /// The CIEs that have been emitted in the output section. The actual CIE
   /// data serves a the key to this StringMap, this takes care of comparing the
@@ -461,6 +473,7 @@ class DwarfLinker {
   uint32_t LastCIEOffset = 0;
 
   /// Apple accelerator tables.
+  AccelTable<DWARF5AccelTableStaticData> DebugNames;
   AccelTable<AppleAccelTableStaticOffsetData> AppleNames;
   AccelTable<AppleAccelTableStaticOffsetData> AppleNamespaces;
   AccelTable<AppleAccelTableStaticOffsetData> AppleObjc;

llvm/tools/dsymutil/DwarfStreamer.cpp

@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DwarfStreamer.h"
+#include "CompileUnit.h"
 #include "LinkUtils.h"
 #include "MachOUtils.h"
 #include "llvm/ADT/Triple.h"
@@ -165,6 +166,9 @@ void DwarfStreamer::emitCompileUnitHeader(CompileUnit &Unit) {
   // start of the section.
   Asm->emitInt32(0);
   Asm->emitInt8(Unit.getOrigUnit().getAddressByteSize());
+
+  // Remember this CU.
+  EmittedUnits.push_back({Unit.getUniqueID(), Unit.getLabelBegin()});
 }
 
 /// Emit the \p Abbrevs array as the shared abbreviation table
@@ -197,6 +201,29 @@ void DwarfStreamer::emitStrings(const NonRelocatableStringpool &Pool) {
   }
 }
 
+void DwarfStreamer::emitDebugNames(
+    AccelTable<DWARF5AccelTableStaticData> &Table) {
+  if (EmittedUnits.empty())
+    return;
+
+  // Build up data structures needed to emit this section.
+  std::vector<MCSymbol *> CompUnits;
+  DenseMap<unsigned, size_t> UniqueIdToCuMap;
+  unsigned Id = 0;
+  for (auto &CU : EmittedUnits) {
+    CompUnits.push_back(CU.LabelBegin);
+    // We might be omitting CUs, so we need to remap them.
+    UniqueIdToCuMap[CU.ID] = Id++;
+  }
+
+  Asm->OutStreamer->SwitchSection(MOFI->getDwarfDebugNamesSection());
+  emitDWARF5AccelTable(
+      Asm.get(), Table, CompUnits,
+      [&UniqueIdToCuMap](const DWARF5AccelTableStaticData &Entry) {
+        return UniqueIdToCuMap[Entry.getCUIndex()];
+      });
+}
+
 void DwarfStreamer::emitAppleNamespaces(
     AccelTable<AppleAccelTableStaticOffsetData> &Table) {
   Asm->OutStreamer->SwitchSection(MOFI->getDwarfAccelNamespaceSection());

llvm/tools/dsymutil/DwarfStreamer.h

@@ -27,6 +27,7 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -121,6 +122,9 @@ class DwarfStreamer {
   void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address,
                StringRef Bytes);
 
+  /// Emit DWARF debug names.
+  void emitDebugNames(AccelTable<DWARF5AccelTableStaticData> &Table);
+
   /// Emit Apple namespaces accelerator table.
   void emitAppleNamespaces(AccelTable<AppleAccelTableStaticOffsetData> &Table);
 
@@ -162,6 +166,13 @@ class DwarfStreamer {
   uint32_t LineSectionSize;
   uint32_t FrameSectionSize;
 
+  /// Keep track of emitted CUs and their Unique ID.
+  struct EmittedUnit {
+    unsigned ID;
+    MCSymbol *LabelBegin;
+  };
+  std::vector<EmittedUnit> EmittedUnits;
+
   /// Emit the pubnames or pubtypes section contribution for \p
   /// Unit into \p Sec. The data is provided in \p Names.
   void emitPubSectionForUnit(MCSection *Sec, StringRef Name,

llvm/tools/dsymutil/LinkUtils.h

@@ -22,6 +22,13 @@ enum class OutputFileType {
   Assembly,
 };
 
+/// The kind of accelerator tables we should emit.
+enum class AccelTableKind {
+  Apple,   ///< .apple_names, .apple_namespaces, .apple_types, .apple_objc.
+  Dwarf,   ///< DWARF v5 .debug_names.
+  Default, ///< Dwarf for DWARF5 or later, Apple otherwise.
+};
+
 struct LinkOptions {
   /// Verbosity
   bool Verbose = false;
@@ -47,6 +54,9 @@ struct LinkOptions {
   // Output file type.
   OutputFileType FileType = OutputFileType::Object;
 
+  /// The accelerator table kind
+  AccelTableKind TheAccelTableKind;
+
   /// -oso-prepend-path
   std::string PrependPath;