[ThinLTO] Always create linked objects file for --thinlto-index-only=

Summary:
ThinLTO indexing may decide to skip all objects. If we don't write something to
the list build system may consider this as failure or linker can reuse a file
from the previews build.

Reviewers: pcc, tejohnson

Subscribers: mehdi_amini, inglorion, eraman, hiraditya, llvm-commits

Differential Revision: https://reviews.llvm.org/D43415

llvm-svn: 325819

llvm/include/llvm/LTO/LTO.h

@@ -210,13 +210,15 @@ ThinBackend createInProcessThinBackend(unsigned ParallelismLevel);
 /// appends ".thinlto.bc" and writes the index to that path. If
 /// ShouldEmitImportsFiles is true it also writes a list of imported files to a
 /// similar path with ".imports" appended instead.
+/// LinkedObjectsFile is an output stream to write the list of object files for
+/// the final ThinLTO linking. Can be nullptr.
 /// OnWrite is callback which receives module identifier and notifies LTO user
 /// that index file for the module (and optionally imports file) was created.
 using IndexWriteCallback = std::function<void(const std::string &)>;
 ThinBackend createWriteIndexesThinBackend(std::string OldPrefix,
                                           std::string NewPrefix,
                                           bool ShouldEmitImportsFiles,
-                                          std::string LinkedObjectsFile,
+                                          raw_fd_ostream *LinkedObjectsFile,
                                           IndexWriteCallback OnWrite);
 
 /// This class implements a resolution-based interface to LLVM's LTO

llvm/lib/LTO/LTO.cpp

@@ -1038,22 +1038,19 @@ namespace {
 class WriteIndexesThinBackend : public ThinBackendProc {
   std::string OldPrefix, NewPrefix;
   bool ShouldEmitImportsFiles;
-
-  std::string LinkedObjectsFileName;
-  std::unique_ptr<llvm::raw_fd_ostream> LinkedObjectsFile;
-
+  raw_fd_ostream *LinkedObjectsFile;
   lto::IndexWriteCallback OnWrite;
 
 public:
   WriteIndexesThinBackend(
       Config &Conf, ModuleSummaryIndex &CombinedIndex,
       const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
       std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
-      std::string LinkedObjectsFileName, lto::IndexWriteCallback OnWrite)
+      raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
       : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
         OldPrefix(OldPrefix), NewPrefix(NewPrefix),
         ShouldEmitImportsFiles(ShouldEmitImportsFiles),
-        LinkedObjectsFileName(LinkedObjectsFileName), OnWrite(OnWrite) {}
+        LinkedObjectsFile(LinkedObjectsFile), OnWrite(OnWrite) {}
 
   Error start(
       unsigned Task, BitcodeModule BM,
@@ -1065,21 +1062,14 @@ class WriteIndexesThinBackend : public ThinBackendProc {
     std::string NewModulePath =
         getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
 
-    std::error_code EC;
-    if (!LinkedObjectsFileName.empty()) {
-      if (!LinkedObjectsFile) {
-        LinkedObjectsFile = llvm::make_unique<raw_fd_ostream>(
-            LinkedObjectsFileName, EC, sys::fs::OpenFlags::F_None);
-        if (EC)
-          return errorCodeToError(EC);
-      }
+    if (LinkedObjectsFile)
       *LinkedObjectsFile << NewModulePath << '\n';
-    }
 
     std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
     gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
                                      ImportList, ModuleToSummariesForIndex);
 
+    std::error_code EC;
     raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
                       sys::fs::OpenFlags::F_None);
     if (EC)
@@ -1101,11 +1091,9 @@ class WriteIndexesThinBackend : public ThinBackendProc {
 };
 } // end anonymous namespace
 
-ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix,
-                                               std::string NewPrefix,
-                                               bool ShouldEmitImportsFiles,
-                                               std::string LinkedObjectsFile,
-                                               IndexWriteCallback OnWrite) {
+ThinBackend lto::createWriteIndexesThinBackend(
+    std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
+    raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
   return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
              const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
              AddStreamFn AddStream, NativeObjectCache Cache) {

llvm/test/tools/gold/X86/thinlto_no_objects.ll

@@ -0,0 +1,18 @@
+; Check that thinlto-index-only= always creates linked objects file, even
+; if nothing to add there.
+
+; Non-ThinLTO file should not get into list of linked objects.
+; RUN: opt %s -o %t.o
+
+; RUN: rm -f %t3
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=thinlto-index-only=%t3 \
+; RUN:    -o %t5 \
+; RUN:    %t.o
+
+; RUN: cat %t3 | count 0
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+

llvm/tools/gold/gold-plugin.cpp

@@ -736,7 +736,12 @@ static void getThinLTOOldAndNewPrefix(std::string &OldPrefix,
   std::tie(OldPrefix, NewPrefix) = PrefixReplace.split(';');
 }
 
-static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite) {
+/// Creates instance of LTO.
+/// OnIndexWrite is callback to let caller know when LTO writes index files.
+/// LinkedObjectsFile is an output stream to write the list of object files for
+/// the final ThinLTO linking. Can be nullptr.
+static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite,
+                                      raw_fd_ostream *LinkedObjectsFile) {
   Config Conf;
   ThinBackend Backend;
 
@@ -760,9 +765,9 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite) {
   if (options::thinlto_index_only) {
     std::string OldPrefix, NewPrefix;
     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
-    Backend = createWriteIndexesThinBackend(
-        OldPrefix, NewPrefix, options::thinlto_emit_imports_files,
-        options::thinlto_linked_objects_file, OnIndexWrite);
+    Backend = createWriteIndexesThinBackend(OldPrefix, NewPrefix,
+                                            options::thinlto_emit_imports_files,
+                                            LinkedObjectsFile, OnIndexWrite);
   }
 
   Conf.OverrideTriple = options::triple;
@@ -844,6 +849,21 @@ static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
   }
 }
 
+// Creates and returns output stream with a list of object files for final
+// linking of distributed ThinLTO.
+static std::unique_ptr<raw_fd_ostream> CreateLinkedObjectsFile() {
+  if (options::thinlto_linked_objects_file.empty())
+    return nullptr;
+  assert(options::thinlto_index_only);
+  std::error_code EC;
+  auto LinkedObjectsFile = llvm::make_unique<raw_fd_ostream>(
+      options::thinlto_linked_objects_file, EC, sys::fs::OpenFlags::F_None);
+  if (EC)
+    message(LDPL_FATAL, "Failed to create '%s': %s",
+            options::thinlto_linked_objects_file.c_str(), EC.message().c_str());
+  return LinkedObjectsFile;
+}
+
 /// Runs LTO and return a list of pairs <FileName, IsTemporary>.
 static std::vector<std::pair<SmallString<128>, bool>> runLTO() {
   // Map to own RAII objects that manage the file opening and releasing
@@ -856,10 +876,12 @@ static std::vector<std::pair<SmallString<128>, bool>> runLTO() {
   // Owns string objects and tells if index file was already created.
   StringMap<bool> ObjectToIndexFileState;
 
-  std::unique_ptr<LTO> Lto =
-      createLTO([&ObjectToIndexFileState](const std::string &Identifier) {
+  std::unique_ptr<raw_fd_ostream> LinkedObjects = CreateLinkedObjectsFile();
+  std::unique_ptr<LTO> Lto = createLTO(
+      [&ObjectToIndexFileState](const std::string &Identifier) {
         ObjectToIndexFileState[Identifier] = true;
-      });
+      },
+      LinkedObjects.get());
 
   std::string OldPrefix, NewPrefix;
   if (options::thinlto_index_only)

llvm/tools/llvm-lto2/llvm-lto2.cpp

@@ -243,7 +243,11 @@ static int run(int argc, char **argv) {
 
   ThinBackend Backend;
   if (ThinLTODistributedIndexes)
-    Backend = createWriteIndexesThinBackend("", "", true, "", {});
+    Backend = createWriteIndexesThinBackend(/* OldPrefix */ "",
+                                            /* NewPrefix */ "",
+                                            /* ShouldEmitImportsFiles */ true,
+                                            /* LinkedObjectsFile */ nullptr,
+                                            /* OnWrite */ {});
   else
     Backend = createInProcessThinBackend(Threads);
   LTO Lto(std::move(Conf), std::move(Backend));