Re-land [LLD][COFF] Early load PDB type server files

We need to have all input files ready before doing debuginfo type merging.
This patch is moving the late PDB type server discovery much earlier in the process, when the explicit inputs (OBJs, LIBs) are loaded.
The short term goal is to parallelize type merging.

Differential Revision: https://reviews.llvm.org/D60095

llvm-svn: 362393

lld/COFF/DebugTypes.cpp

@@ -7,38 +7,81 @@
 //===----------------------------------------------------------------------===//
 
 #include "DebugTypes.h"
+#include "Driver.h"
 #include "InputFiles.h"
+#include "lld/Common/ErrorHandler.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/PDB/GenericError.h"
+#include "llvm/DebugInfo/PDB/Native/InfoStream.h"
+#include "llvm/DebugInfo/PDB/Native/NativeSession.h"
+#include "llvm/DebugInfo/PDB/Native/PDBFile.h"
+#include "llvm/Support/Path.h"
 
 using namespace lld;
 using namespace lld::coff;
 using namespace llvm;
 using namespace llvm::codeview;
 
 namespace {
+// The TypeServerSource class represents a PDB type server, a file referenced by
+// OBJ files compiled with MSVC /Zi. A single PDB can be shared by several OBJ
+// files, therefore there must be only once instance per OBJ lot. The file path
+// is discovered from the dependent OBJ's debug type stream. The
+// TypeServerSource object is then queued and loaded by the COFF Driver. The
+// debug type stream for such PDB files will be merged first in the final PDB,
+// before any dependent OBJ.
 class TypeServerSource : public TpiSource {
 public:
-  TypeServerSource(ObjFile *F) : TpiSource(PDB, F) {}
+  explicit TypeServerSource(MemoryBufferRef M, llvm::pdb::NativeSession *S)
+      : TpiSource(PDB, nullptr), Session(S), MB(M) {}
+
+  // Queue a PDB type server for loading in the COFF Driver
+  static void enqueue(const ObjFile *DependentFile,
+                      const TypeServer2Record &TS);
+
+  // Create an instance
+  static Expected<TypeServerSource *> getInstance(MemoryBufferRef M);
+
+  // Fetch the PDB instance loaded for a corresponding dependent OBJ.
+  static Expected<TypeServerSource *>
+  findFromFile(const ObjFile *DependentFile);
+
+  static std::map<std::string, std::pair<std::string, TypeServerSource *>>
+      Instances;
+
+  // The interface to the PDB (if it was opened successfully)
+  std::unique_ptr<llvm::pdb::NativeSession> Session;
+
+private:
+  MemoryBufferRef MB;
 };
 
+// This class represents the debug type stream of an OBJ file that depends on a
+// PDB type server (see TypeServerSource).
 class UseTypeServerSource : public TpiSource {
 public:
-  UseTypeServerSource(ObjFile *F, TypeServer2Record *TS)
+  UseTypeServerSource(const ObjFile *F, const TypeServer2Record *TS)
       : TpiSource(UsingPDB, F), TypeServerDependency(*TS) {}
 
   // Information about the PDB type server dependency, that needs to be loaded
   // in before merging this OBJ.
   TypeServer2Record TypeServerDependency;
 };
 
+// This class represents the debug type stream of a Microsoft precompiled
+// headers OBJ (PCH OBJ). This OBJ kind needs to be merged first in the output
+// PDB, before any other OBJs that depend on this. Note that only MSVC generate
+// such files, clang does not.
 class PrecompSource : public TpiSource {
 public:
-  PrecompSource(ObjFile *F) : TpiSource(PCH, F) {}
+  PrecompSource(const ObjFile *F) : TpiSource(PCH, F) {}
 };
 
+// This class represents the debug type stream of an OBJ file that depends on a
+// Microsoft precompiled headers OBJ (see PrecompSource).
 class UsePrecompSource : public TpiSource {
 public:
-  UsePrecompSource(ObjFile *F, PrecompRecord *Precomp)
+  UsePrecompSource(const ObjFile *F, const PrecompRecord *Precomp)
       : TpiSource(UsingPCH, F), PrecompDependency(*Precomp) {}
 
   // Information about the Precomp OBJ dependency, that needs to be loaded in
@@ -49,40 +92,177 @@ class UsePrecompSource : public TpiSource {
 
 static std::vector<std::unique_ptr<TpiSource>> GC;
 
-TpiSource::TpiSource(TpiKind K, ObjFile *F) : Kind(K), File(F) {
+TpiSource::TpiSource(TpiKind K, const ObjFile *F) : Kind(K), File(F) {
   GC.push_back(std::unique_ptr<TpiSource>(this));
 }
 
-TpiSource *coff::makeTpiSource(ObjFile *F) {
+TpiSource *lld::coff::makeTpiSource(const ObjFile *F) {
   return new TpiSource(TpiSource::Regular, F);
 }
 
-TpiSource *coff::makeTypeServerSource(ObjFile *F) {
-  return new TypeServerSource(F);
-}
-
-TpiSource *coff::makeUseTypeServerSource(ObjFile *F, TypeServer2Record *TS) {
+TpiSource *lld::coff::makeUseTypeServerSource(const ObjFile *F,
+                                              const TypeServer2Record *TS) {
+  TypeServerSource::enqueue(F, *TS);
   return new UseTypeServerSource(F, TS);
 }
 
-TpiSource *coff::makePrecompSource(ObjFile *F) { return new PrecompSource(F); }
+TpiSource *lld::coff::makePrecompSource(const ObjFile *F) {
+  return new PrecompSource(F);
+}
 
-TpiSource *coff::makeUsePrecompSource(ObjFile *F, PrecompRecord *Precomp) {
+TpiSource *lld::coff::makeUsePrecompSource(const ObjFile *F,
+                                           const PrecompRecord *Precomp) {
   return new UsePrecompSource(F, Precomp);
 }
 
 namespace lld {
 namespace coff {
 template <>
-const PrecompRecord &retrieveDependencyInfo(TpiSource *Source) {
+const PrecompRecord &retrieveDependencyInfo(const TpiSource *Source) {
   assert(Source->Kind == TpiSource::UsingPCH);
-  return ((UsePrecompSource *)Source)->PrecompDependency;
+  return ((const UsePrecompSource *)Source)->PrecompDependency;
 }
 
 template <>
-const TypeServer2Record &retrieveDependencyInfo(TpiSource *Source) {
+const TypeServer2Record &retrieveDependencyInfo(const TpiSource *Source) {
   assert(Source->Kind == TpiSource::UsingPDB);
-  return ((UseTypeServerSource *)Source)->TypeServerDependency;
+  return ((const UseTypeServerSource *)Source)->TypeServerDependency;
 }
 } // namespace coff
 } // namespace lld
+
+std::map<std::string, std::pair<std::string, TypeServerSource *>>
+    TypeServerSource::Instances;
+
+// Make a PDB path assuming the PDB is in the same folder as the OBJ
+static std::string getPdbBaseName(const ObjFile *File, StringRef TSPath) {
+  StringRef LocalPath =
+      !File->ParentName.empty() ? File->ParentName : File->getName();
+  SmallString<128> Path = sys::path::parent_path(LocalPath);
+
+  // Currently, type server PDBs are only created by MSVC cl, which only runs
+  // on Windows, so we can assume type server paths are Windows style.
+  sys::path::append(Path, sys::path::filename(TSPath, sys::path::Style::windows));
+  return Path.str();
+}
+
+// The casing of the PDB path stamped in the OBJ can differ from the actual path
+// on disk. With this, we ensure to always use lowercase as a key for the
+// PDBInputFile::Instances map, at least on Windows.
+static std::string normalizePdbPath(StringRef path) {
+#if defined(_WIN32)
+  return path.lower();
+#else // LINUX
+  return path;
+#endif
+}
+
+// If existing, return the actual PDB path on disk.
+static Optional<std::string> findPdbPath(StringRef PDBPath,
+                                         const ObjFile *DependentFile) {
+  // Ensure the file exists before anything else. In some cases, if the path
+  // points to a removable device, Driver::enqueuePath() would fail with an
+  // error (EAGAIN, "resource unavailable try again") which we want to skip
+  // silently.
+  if (llvm::sys::fs::exists(PDBPath))
+    return normalizePdbPath(PDBPath);
+  std::string Ret = getPdbBaseName(DependentFile, PDBPath);
+  if (llvm::sys::fs::exists(Ret))
+    return normalizePdbPath(Ret);
+  return None;
+}
+
+// Fetch the PDB instance that was already loaded by the COFF Driver.
+Expected<TypeServerSource *>
+TypeServerSource::findFromFile(const ObjFile *DependentFile) {
+  const TypeServer2Record &TS =
+      retrieveDependencyInfo<TypeServer2Record>(DependentFile->DebugTypesObj);
+
+  Optional<std::string> P = findPdbPath(TS.Name, DependentFile);
+  if (!P)
+    return createFileError(TS.Name, errorCodeToError(std::error_code(
+                                        ENOENT, std::generic_category())));
+
+  auto It = TypeServerSource::Instances.find(*P);
+  // The PDB file exists on disk, at this point we expect it to have been
+  // inserted in the map by TypeServerSource::loadPDB()
+  assert(It != TypeServerSource::Instances.end());
+
+  std::pair<std::string, TypeServerSource *> &PDB = It->second;
+
+  if (!PDB.second)
+    return createFileError(
+        *P, createStringError(inconvertibleErrorCode(), PDB.first.c_str()));
+
+  pdb::PDBFile &PDBFile = (PDB.second)->Session->getPDBFile();
+  pdb::InfoStream &Info = cantFail(PDBFile.getPDBInfoStream());
+
+  // Just because a file with a matching name was found doesn't mean it can be
+  // used. The GUID and Age must match between the PDB header and the OBJ
+  // TypeServer2 record. The 'Age' is used by MSVC incremental compilation.
+  if (Info.getGuid() != TS.getGuid() || Info.getAge() != TS.getAge())
+    return createFileError(
+        TS.Name,
+        make_error<pdb::PDBError>(pdb::pdb_error_code::signature_out_of_date));
+
+  return PDB.second;
+}
+
+// FIXME: Temporary interface until PDBLinker::maybeMergeTypeServerPDB() is
+// moved here.
+Expected<llvm::pdb::NativeSession *>
+lld::coff::findTypeServerSource(const ObjFile *F) {
+  Expected<TypeServerSource *> TS = TypeServerSource::findFromFile(F);
+  if (!TS)
+    return TS.takeError();
+  return TS.get()->Session.get();
+}
+
+// Queue a PDB type server for loading in the COFF Driver
+void TypeServerSource::enqueue(const ObjFile *DependentFile,
+                               const TypeServer2Record &TS) {
+  // Start by finding where the PDB is located (either the record path or next
+  // to the OBJ file)
+  Optional<std::string> P = findPdbPath(TS.Name, DependentFile);
+  if (!P)
+    return;
+  auto It = TypeServerSource::Instances.emplace(
+      *P, std::pair<std::string, TypeServerSource *>{});
+  if (!It.second)
+    return; // another OBJ already scheduled this PDB for load
+
+  Driver->enqueuePath(*P, false);
+}
+
+// Create an instance of TypeServerSource or an error string if the PDB couldn't
+// be loaded. The error message will be displayed later, when the referring OBJ
+// will be merged in. NOTE - a PDB load failure is not a link error: some
+// debug info will simply be missing from the final PDB - that is the default
+// accepted behavior.
+void lld::coff::loadTypeServerSource(llvm::MemoryBufferRef M) {
+  std::string Path = normalizePdbPath(M.getBufferIdentifier());
+
+  Expected<TypeServerSource *> TS = TypeServerSource::getInstance(M);
+  if (!TS)
+    TypeServerSource::Instances[Path] = {toString(TS.takeError()), nullptr};
+  else
+    TypeServerSource::Instances[Path] = {{}, *TS};
+}
+
+Expected<TypeServerSource *> TypeServerSource::getInstance(MemoryBufferRef M) {
+  std::unique_ptr<llvm::pdb::IPDBSession> ISession;
+  Error Err = pdb::NativeSession::createFromPdb(
+      MemoryBuffer::getMemBuffer(M, false), ISession);
+  if (Err)
+    return std::move(Err);
+
+  std::unique_ptr<llvm::pdb::NativeSession> Session(
+      static_cast<pdb::NativeSession *>(ISession.release()));
+
+  pdb::PDBFile &PDBFile = Session->getPDBFile();
+  Expected<pdb::InfoStream &> Info = PDBFile.getPDBInfoStream();
+  // All PDB Files should have an Info stream.
+  if (!Info)
+    return Info.takeError();
+  return new TypeServerSource(M, Session.release());
+}

lld/COFF/DebugTypes.h

@@ -10,12 +10,16 @@
 #define LLD_COFF_DEBUGTYPES_H
 
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 
 namespace llvm {
 namespace codeview {
 class PrecompRecord;
 class TypeServer2Record;
 } // namespace codeview
+namespace pdb {
+class NativeSession;
+}
 } // namespace llvm
 
 namespace lld {
@@ -27,23 +31,28 @@ class TpiSource {
 public:
   enum TpiKind { Regular, PCH, UsingPCH, PDB, UsingPDB };
 
-  TpiSource(TpiKind K, ObjFile *F);
+  TpiSource(TpiKind K, const ObjFile *F);
   virtual ~TpiSource() {}
 
   const TpiKind Kind;
-  ObjFile *File;
+  const ObjFile *File;
 };
 
-TpiSource *makeTpiSource(ObjFile *F);
-TpiSource *makeTypeServerSource(ObjFile *F);
-TpiSource *makeUseTypeServerSource(ObjFile *F,
-                                   llvm::codeview::TypeServer2Record *TS);
-TpiSource *makePrecompSource(ObjFile *F);
-TpiSource *makeUsePrecompSource(ObjFile *F,
-                                llvm::codeview::PrecompRecord *Precomp);
+TpiSource *makeTpiSource(const ObjFile *F);
+TpiSource *makeUseTypeServerSource(const ObjFile *F,
+                                   const llvm::codeview::TypeServer2Record *TS);
+TpiSource *makePrecompSource(const ObjFile *F);
+TpiSource *makeUsePrecompSource(const ObjFile *F,
+                                const llvm::codeview::PrecompRecord *Precomp);
+
+void loadTypeServerSource(llvm::MemoryBufferRef M);
 
 // Temporary interface to get the dependency
-template <typename T> const T &retrieveDependencyInfo(TpiSource *Source);
+template <typename T> const T &retrieveDependencyInfo(const TpiSource *Source);
+
+// Temporary interface until we move PDBLinker::maybeMergeTypeServerPDB here
+llvm::Expected<llvm::pdb::NativeSession *>
+findTypeServerSource(const ObjFile *F);
 
 } // namespace coff
 } // namespace lld

lld/COFF/Driver.cpp

@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "DebugTypes.h"
 #include "Driver.h"
 #include "Config.h"
 #include "ICF.h"
@@ -181,6 +182,9 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> MB,
   case file_magic::coff_import_library:
     Symtab->addFile(make<ObjFile>(MBRef));
     break;
+  case file_magic::pdb:
+    loadTypeServerSource(MBRef);
+    break;
   case file_magic::coff_cl_gl_object:
     error(Filename + ": is not a native COFF file. Recompile without /GL");
     break;

lld/COFF/Driver.h

@@ -77,6 +77,8 @@ class LinkerDriver {
 
   MemoryBufferRef takeBuffer(std::unique_ptr<MemoryBuffer> MB);
 
+  void enqueuePath(StringRef Path, bool WholeArchive);
+
 private:
   std::unique_ptr<llvm::TarWriter> Tar; // for /linkrepro
 
@@ -120,8 +122,6 @@ class LinkerDriver {
   void addArchiveBuffer(MemoryBufferRef MBRef, StringRef SymName,
                         StringRef ParentName, uint64_t OffsetInArchive);
 
-  void enqueuePath(StringRef Path, bool WholeArchive);
-
   void enqueueTask(std::function<void()> Task);
   bool run();