refactor COFF linker to use new LTO API

Summary: The COFF linker previously implemented link-time optimization using an API which has now been marked as legacy. This change refactors the COFF linker to use the new LTO API, which is also used by the ELF linker.

Reviewers: pcc, ruiu

Reviewed By: pcc

Subscribers: mgorny, mehdi_amini

Differential Revision: https://reviews.llvm.org/D29059

llvm-svn: 293967

Author: inglorion

lld/COFF/CMakeLists.txt

@@ -15,6 +15,7 @@ add_lld_library(lldCOFF
   ICF.cpp
   InputFiles.cpp
   Librarian.cpp
+  LTO.cpp
   MapFile.cpp
   MarkLive.cpp
   ModuleDef.cpp

lld/COFF/Config.h

@@ -114,6 +114,8 @@ struct Configuration {
 
   // Used for /opt:lldltojobs=N
   unsigned LTOJobs = 1;
+  // Used for /opt:lldltopartitions=N
+  unsigned LTOPartitions = 1;
 
   // Used for /merge:from=to (e.g. /merge:.rdata=.text)
   std::map<StringRef, StringRef> Merge;

lld/COFF/Driver.cpp

@@ -609,6 +609,13 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
           fatal("/opt:lldltojobs: invalid job count: " + Jobs);
         continue;
       }
+      if (StringRef(S).startswith("lldltopartitions=")) {
+        StringRef N = StringRef(S).substr(17);
+        if (N.getAsInteger(10, Config->LTOPartitions) ||
+            Config->LTOPartitions == 0)
+          fatal("/opt:lldltopartitions: invalid partition count: " + N);
+        continue;
+      }
       if (S != "ref" && S != "lbr" && S != "nolbr")
         fatal("/opt: unknown option: " + S);
     }

lld/COFF/InputFiles.cpp

@@ -19,8 +19,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/LTO/legacy/LTOModule.h"
+#include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Support/COFF.h"
@@ -45,7 +44,19 @@ using llvm::support::ulittle32_t;
 namespace lld {
 namespace coff {
 
-LLVMContext BitcodeFile::Context;
+/// Checks that Source is compatible with being a weak alias to Target.
+/// If Source is Undefined and has no weak alias set, makes it a weak
+/// alias to Target.
+static void checkAndSetWeakAlias(SymbolTable *Symtab, InputFile *F,
+                                 SymbolBody *Source, SymbolBody *Target) {
+  auto *U = dyn_cast<Undefined>(Source);
+  if (!U)
+    return;
+  else if (!U->WeakAlias)
+    U->WeakAlias = Target;
+  else if (U->WeakAlias != Target)
+    Symtab->reportDuplicate(Source->symbol(), F);
+}
 
 ArchiveFile::ArchiveFile(MemoryBufferRef M) : InputFile(ArchiveKind, M) {}
 
@@ -175,15 +186,9 @@ void ObjectFile::initializeSymbols() {
     I += Sym.getNumberOfAuxSymbols();
     LastSectionNumber = Sym.getSectionNumber();
   }
-  for (auto WeakAlias : WeakAliases) {
-    auto *U = dyn_cast<Undefined>(WeakAlias.first);
-    if (!U)
-      continue;
-    // Report an error if two undefined symbols have different weak aliases.
-    if (U->WeakAlias && U->WeakAlias != SparseSymbolBodies[WeakAlias.second])
-      Symtab->reportDuplicate(U->symbol(), this);
-    U->WeakAlias = SparseSymbolBodies[WeakAlias.second];
-  }
+  for (auto WeakAlias : WeakAliases)
+    checkAndSetWeakAlias(Symtab, this, WeakAlias.first,
+                         SparseSymbolBodies[WeakAlias.second]);
 }
 
 SymbolBody *ObjectFile::createUndefined(COFFSymbolRef Sym) {
@@ -198,7 +203,10 @@ SymbolBody *ObjectFile::createDefined(COFFSymbolRef Sym, const void *AuxP,
   if (Sym.isCommon()) {
     auto *C = new (Alloc) CommonChunk(Sym);
     Chunks.push_back(C);
-    return Symtab->addCommon(this, Sym, C)->body();
+    COFFObj->getSymbolName(Sym, Name);
+    Symbol *S =
+        Symtab->addCommon(this, Name, Sym.getValue(), Sym.getGeneric(), C);
+    return S->body();
   }
   if (Sym.isAbsolute()) {
     COFFObj->getSymbolName(Sym, Name);
@@ -245,10 +253,14 @@ SymbolBody *ObjectFile::createDefined(COFFSymbolRef Sym, const void *AuxP,
   }
 
   DefinedRegular *B;
-  if (Sym.isExternal())
-    B = cast<DefinedRegular>(Symtab->addRegular(this, Sym, SC)->body());
-  else
-    B = new (Alloc) DefinedRegular(this, Sym, SC);
+  if (Sym.isExternal()) {
+    COFFObj->getSymbolName(Sym, Name);
+    Symbol *S =
+        Symtab->addRegular(this, Name, SC->isCOMDAT(), Sym.getGeneric(), SC);
+    B = cast<DefinedRegular>(S->body());
+  } else
+    B = new (Alloc) DefinedRegular(this, /*Name*/ "", SC->isCOMDAT(),
+                                   /*IsExternal*/ false, Sym.getGeneric(), SC);
   if (SC->isCOMDAT() && Sym.getValue() == 0 && !AuxP)
     SC->setSymbol(B);
 
@@ -327,39 +339,38 @@ void ImportFile::parse() {
 }
 
 void BitcodeFile::parse() {
-  Context.enableDebugTypeODRUniquing();
-  ErrorOr<std::unique_ptr<LTOModule>> ModOrErr = LTOModule::createFromBuffer(
-      Context, MB.getBufferStart(), MB.getBufferSize(), llvm::TargetOptions());
-  M = check(std::move(ModOrErr), "could not create LTO module");
-
-  StringSaver Saver(Alloc);
-  for (unsigned I = 0, E = M->getSymbolCount(); I != E; ++I) {
-    lto_symbol_attributes Attrs = M->getSymbolAttributes(I);
-    if ((Attrs & LTO_SYMBOL_SCOPE_MASK) == LTO_SYMBOL_SCOPE_INTERNAL)
-      continue;
-
-    StringRef SymName = Saver.save(M->getSymbolName(I));
-    int SymbolDef = Attrs & LTO_SYMBOL_DEFINITION_MASK;
-    if (SymbolDef == LTO_SYMBOL_DEFINITION_UNDEFINED) {
-      SymbolBodies.push_back(Symtab->addUndefined(SymName, this, false)->body());
+  Obj = check(lto::InputFile::create(
+      MemoryBufferRef(MB.getBuffer(), Saver.save(MB.getBufferIdentifier()))));
+  for (const lto::InputFile::Symbol &ObjSym : Obj->symbols()) {
+    StringRef SymName = Saver.save(ObjSym.getName());
+    auto Flags = ObjSym.getFlags();
+    Symbol *Sym;
+    if (Flags & object::BasicSymbolRef::SF_Undefined) {
+      Sym = Symtab->addUndefined(SymName, this, false);
+    } else if (Flags & object::BasicSymbolRef::SF_Common) {
+      Sym = Symtab->addCommon(this, SymName, ObjSym.getCommonSize());
+    } else if ((Flags & object::BasicSymbolRef::SF_Weak) &&
+               (Flags & object::BasicSymbolRef::SF_Indirect)) {
+      // Weak external.
+      Sym = Symtab->addUndefined(SymName, this, true);
+      std::string Fallback = ObjSym.getCOFFWeakExternalFallback();
+      SymbolBody *Alias = Symtab->addUndefined(Saver.save(Fallback));
+      checkAndSetWeakAlias(Symtab, this, Sym->body(), Alias);
     } else {
-      bool Replaceable =
-          (SymbolDef == LTO_SYMBOL_DEFINITION_TENTATIVE || // common
-           (Attrs & LTO_SYMBOL_COMDAT) ||                  // comdat
-           (SymbolDef == LTO_SYMBOL_DEFINITION_WEAK &&     // weak external
-            (Attrs & LTO_SYMBOL_ALIAS)));
-      SymbolBodies.push_back(
-          Symtab->addBitcode(this, SymName, Replaceable)->body());
+      Expected<int> ComdatIndex = ObjSym.getComdatIndex();
+      bool IsCOMDAT = ComdatIndex && *ComdatIndex != -1;
+      Sym = Symtab->addRegular(this, SymName, IsCOMDAT);
     }
+    SymbolBodies.push_back(Sym->body());
   }
-
-  Directives = M->getLinkerOpts();
+  Directives = check(Obj->getLinkerOpts());
 }
 
 MachineTypes BitcodeFile::getMachineType() {
-  if (!M)
+  Expected<std::string> ET = getBitcodeTargetTriple(MB);
+  if (!ET)
     return IMAGE_FILE_MACHINE_UNKNOWN;
-  switch (Triple(M->getTargetTriple()).getArch()) {
+  switch (Triple(*ET).getArch()) {
   case Triple::x86_64:
     return AMD64;
   case Triple::x86:

lld/COFF/InputFiles.h

@@ -13,8 +13,7 @@
 #include "lld/Core/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/LTO/legacy/LTOModule.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Support/StringSaver.h"
@@ -25,7 +24,6 @@
 namespace lld {
 namespace coff {
 
-using llvm::LTOModule;
 using llvm::COFF::IMAGE_FILE_MACHINE_UNKNOWN;
 using llvm::COFF::MachineTypes;
 using llvm::object::Archive;
@@ -191,16 +189,13 @@ class BitcodeFile : public InputFile {
   static bool classof(const InputFile *F) { return F->kind() == BitcodeKind; }
   std::vector<SymbolBody *> &getSymbols() { return SymbolBodies; }
   MachineTypes getMachineType() override;
-  std::unique_ptr<LTOModule> takeModule() { return std::move(M); }
-
-  static llvm::LLVMContext Context;
+  std::unique_ptr<llvm::lto::InputFile> Obj;
 
 private:
   void parse() override;
 
   std::vector<SymbolBody *> SymbolBodies;
   llvm::BumpPtrAllocator Alloc;
-  std::unique_ptr<LTOModule> M;
 };
 } // namespace coff
 

lld/COFF/LTO.cpp

@@ -0,0 +1,125 @@
+//===- LTO.cpp ------------------------------------------------------------===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "LTO.h"
+#include "Config.h"
+#include "Error.h"
+#include "InputFiles.h"
+#include "Symbols.h"
+#include "lld/Core/TargetOptionsCommandFlags.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/LTO/Config.h"
+#include "llvm/LTO/LTO.h"
+#include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <system_error>
+#include <vector>
+
+using namespace llvm;
+using namespace llvm::object;
+
+using namespace lld;
+using namespace lld::coff;
+
+static void diagnosticHandler(const DiagnosticInfo &DI) {
+  SmallString<128> ErrStorage;
+  raw_svector_ostream OS(ErrStorage);
+  DiagnosticPrinterRawOStream DP(OS);
+  DI.print(DP);
+  warn(ErrStorage);
+}
+
+static void checkError(Error E) {
+  handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) -> Error {
+    error(EIB.message());
+    return Error::success();
+  });
+}
+
+static std::unique_ptr<lto::LTO> createLTO() {
+  lto::Config Conf;
+  Conf.Options = InitTargetOptionsFromCodeGenFlags();
+  Conf.RelocModel = Reloc::PIC_;
+  Conf.DisableVerify = true;
+  Conf.DiagHandler = diagnosticHandler;
+  Conf.OptLevel = Config->LTOOptLevel;
+  lto::ThinBackend Backend;
+  if (Config->LTOJobs != -1u)
+    Backend = lto::createInProcessThinBackend(Config->LTOJobs);
+  return llvm::make_unique<lto::LTO>(std::move(Conf), Backend,
+                                     Config->LTOPartitions);
+}
+
+BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) {}
+
+BitcodeCompiler::~BitcodeCompiler() = default;
+
+static void undefine(Symbol *S) {
+  replaceBody<Undefined>(S, S->body()->getName());
+}
+
+void BitcodeCompiler::add(BitcodeFile &F) {
+  lto::InputFile &Obj = *F.Obj;
+  unsigned SymNum = 0;
+  std::vector<SymbolBody *> SymBodies = F.getSymbols();
+  std::vector<lto::SymbolResolution> Resols(SymBodies.size());
+
+  // Provide a resolution to the LTO API for each symbol.
+  for (const lto::InputFile::Symbol &ObjSym : Obj.symbols()) {
+    SymbolBody *B = SymBodies[SymNum];
+    Symbol *Sym = B->symbol();
+    lto::SymbolResolution &R = Resols[SymNum];
+    ++SymNum;
+
+    // Ideally we shouldn't check for SF_Undefined but currently IRObjectFile
+    // reports two symbols for module ASM defined. Without this check, lld
+    // flags an undefined in IR with a definition in ASM as prevailing.
+    // Once IRObjectFile is fixed to report only one symbol this hack can
+    // be removed.
+    R.Prevailing =
+        !(ObjSym.getFlags() & object::BasicSymbolRef::SF_Undefined) &&
+        B->getFile() == &F;
+    R.VisibleToRegularObj = Sym->IsUsedInRegularObj;
+    if (R.Prevailing)
+      undefine(Sym);
+  }
+  checkError(LTOObj->add(std::move(F.Obj), Resols));
+}
+
+// Merge all the bitcode files we have seen, codegen the result
+// and return the resulting ObjectFile(s).
+std::vector<InputFile *> BitcodeCompiler::compile() {
+  std::vector<InputFile *> Ret;
+  unsigned MaxTasks = LTOObj->getMaxTasks();
+  Buff.resize(MaxTasks);
+
+  checkError(LTOObj->run([&](size_t Task) {
+    return llvm::make_unique<lto::NativeObjectStream>(
+        llvm::make_unique<raw_svector_ostream>(Buff[Task]));
+  }));
+
+  for (unsigned I = 0; I != MaxTasks; ++I) {
+    if (Buff[I].empty())
+      continue;
+    Ret.push_back(make<ObjectFile>(MemoryBufferRef(Buff[I], "lto.tmp")));
+  }
+  return Ret;
+}

lld/COFF/LTO.h

@@ -0,0 +1,56 @@
+//===- LTO.h ----------------------------------------------------*- C++ -*-===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides a way to combine bitcode files into one COFF
+// file by compiling them using LLVM.
+//
+// If LTO is in use, your input files are not in regular COFF files
+// but instead LLVM bitcode files. In that case, the linker has to
+// convert bitcode files into the native format so that we can create
+// a COFF file that contains native code. This file provides that
+// functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COFF_LTO_H
+#define LLD_COFF_LTO_H
+
+#include "lld/Core/LLVM.h"
+#include "llvm/ADT/SmallString.h"
+#include <memory>
+#include <vector>
+
+namespace llvm {
+namespace lto {
+class LTO;
+}
+}
+
+namespace lld {
+namespace coff {
+
+class BitcodeFile;
+class InputFile;
+
+class BitcodeCompiler {
+public:
+  BitcodeCompiler();
+  ~BitcodeCompiler();
+
+  void add(BitcodeFile &F);
+  std::vector<InputFile *> compile();
+
+private:
+  std::unique_ptr<llvm::lto::LTO> LTOObj;
+  std::vector<SmallString<0>> Buff;
+};
+}
+}
+
+#endif