[DTLTO][Clang][LLD] Fix DTLTO for multi-call LLVM driver toolchain

clang/include/clang/Driver/CommonArgs.h

@@ -76,6 +76,9 @@ void SplitDebugInfo(const ToolChain &TC, Compilation &C, const Tool &T,
                     const JobAction &JA, const llvm::opt::ArgList &Args,
                     const InputInfo &Output, const char *OutFile);
 
+void addDTLTOOptions(const ToolChain &ToolChain, const llvm::opt::ArgList &Args,
+                     llvm::opt::ArgStringList &CmdArgs);
+
 void addLTOOptions(const ToolChain &ToolChain, const llvm::opt::ArgList &Args,
                    llvm::opt::ArgStringList &CmdArgs, const InputInfo &Output,
                    const InputInfoList &Inputs, bool IsThinLTO);

clang/lib/Driver/ToolChains/CommonArgs.cpp

@@ -949,6 +949,24 @@ bool tools::isTLSDESCEnabled(const ToolChain &TC,
   return EnableTLSDESC;
 }
 
+void tools::addDTLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
+                            llvm::opt::ArgStringList &CmdArgs) {
+  if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) {
+    CmdArgs.push_back(
+        Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue())));
+    const Driver &D = ToolChain.getDriver();
+    CmdArgs.push_back(Args.MakeArgString("--thinlto-remote-compiler=" +
+                                         Twine(D.getClangProgramPath())));
+    if (auto *PA = D.getPrependArg())
+      CmdArgs.push_back(Args.MakeArgString(
+          "--thinlto-remote-compiler-prepend-arg=" + Twine(PA)));
+
+    for (const auto &A :
+         Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ))
+      CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A));
+  }
+}
+
 void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
                           ArgStringList &CmdArgs, const InputInfo &Output,
                           const InputInfoList &Inputs, bool IsThinLTO) {
@@ -1345,16 +1363,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
     CmdArgs.push_back(
         Args.MakeArgString(Twine(PluginOptPrefix) + "-time-passes"));
 
-  if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) {
-    CmdArgs.push_back(
-        Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue())));
-    CmdArgs.push_back(
-        Args.MakeArgString("--thinlto-remote-compiler=" +
-                           Twine(ToolChain.getDriver().getClangProgramPath())));
-
-    for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ))
-      CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A));
-  }
+  addDTLTOOptions(ToolChain, Args, CmdArgs);
 }
 
 void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC,

clang/lib/Driver/ToolChains/PS4CPU.cpp

@@ -344,16 +344,7 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // pass LTO options to ensure proper codegen, metadata production, etc if
   // LTO indeed occurs.
 
-  if (const Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) {
-    CmdArgs.push_back(
-        Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue())));
-    CmdArgs.push_back(Args.MakeArgString("--thinlto-remote-compiler=" +
-                                         Twine(D.getClangProgramPath())));
-
-    for (const auto &A :
-         Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ))
-      CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A));
-  }
+  tools::addDTLTOOptions(TC, Args, CmdArgs);
 
   if (Args.hasFlag(options::OPT_funified_lto, options::OPT_fno_unified_lto,
                    true))

clang/test/Driver/DTLTO/dtlto-helper.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+import sys
+
+# Arg 1: "clang" path.
+p = Path(sys.argv[1])
+print(f"clang-name:{p.resolve().name}")
+# Arg 2: Non-zero for LLVM driver.
+if sys.argv[2] != "0":
+    print(f'prepend-arg:"--thinlto-remote-compiler-prepend-arg={p.name}"')
+else:
+    print("prepend-arg: ")

clang/test/Driver/DTLTO/dtlto.c

@@ -3,16 +3,18 @@
 /// Check DTLTO options are forwarded to the linker.
 
 /// Check that options are forwarded as expected with --thinlto-distributor=.
-// RUN: %python %S/filename.py %clang > %t_forward.log
+// RUN: %python %S/dtlto-helper.py %clang %llvm-driver > %t_forward.log
 // RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \
 // RUN:   -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 \
 // RUN:   -fthinlto-distributor=d.exe -Werror >>%t_forward.log 2>&1
 // RUN: FileCheck %s --input-file=%t_forward.log --check-prefix=FORWARD
 
-// FORWARD: filename.py:[[CLANG:.*]]
-// FORWARD: ld.lld
+// FORWARD:      clang-name:[[CLANG:.*]]
+// FORWARD-NEXT: prepend-arg:[[PREPEND_ARG:.*]]
+// FORWARD:      ld.lld
 // FORWARD-SAME: "--thinlto-distributor=d.exe"
 // FORWARD-SAME: "--thinlto-remote-compiler={{[^"]*}}[[CLANG]]"
+// FORWARD-SAME: [[PREPEND_ARG]]
 // FORWARD-SAME: "--thinlto-distributor-arg=a1"
 // FORWARD-SAME: "--thinlto-distributor-arg=a2"
 // FORWARD-SAME: "--thinlto-distributor-arg=a3"
@@ -30,24 +32,25 @@
 
 /// Check the expected arguments are forwarded by default with only
 /// --thinlto-distributor=.
-// RUN: %python %S/filename.py %clang > %t_default.log
+// RUN: %python %S/dtlto-helper.py %clang %llvm-driver > %t_default.log
 // RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \
 // RUN:   -fthinlto-distributor=d.exe -Werror >>%t_default.log 2>&1
 // RUN: FileCheck %s --input-file=%t_default.log --check-prefix=DEFAULT \
 // RUN:   --implicit-check-not=distributor --implicit-check-not=remote-compiler
 
-// DEFAULT: filename.py:[[CLANG:.*]]
-// DEFAULT: ld.lld
+// DEFAULT:      clang-name:[[CLANG:.*]]
+// DEFAULT-NEXT: prepend-arg:[[PREPEND_ARG:.*]]
+// DEFAULT:      ld.lld
 // DEFAULT-SAME: "--thinlto-distributor=d.exe"
 // DEFAULT-SAME: "--thinlto-remote-compiler={{[^"]*}}[[CLANG]]"
+// DEFAULT-SAME: [[PREPEND_ARG]]
 
 /// Check that nothing is forwarded when the compiler is not in LTO mode, and that
 /// appropriate unused option warnings are issued.
-// RUN: %python %S/filename.py %clang > %t_noflto.log
 // RUN: %clang %s -### -fuse-ld=lld --target=x86_64-linux-gnu \
-// RUN:   -fthinlto-distributor=d.exe  >>%t_noflto.log 2>&1
-// RUN: FileCheck %s --input-file=%t_noflto.log --check-prefix=NOFLTO \
-// RUN:   --implicit-check-not=distributor --implicit-check-not=remote-compiler
+// RUN:   -fthinlto-distributor=d.exe 2>&1 | \
+// RUN: FileCheck %s --check-prefix=NOFLTO --implicit-check-not=distributor \
+// RUN:   --implicit-check-not=remote-compiler
 
 // NOFLTO: warning: argument unused during compilation: '-fthinlto-distributor=d.exe'
 // NOFLTO: ld.lld

clang/test/Driver/DTLTO/lit.local.cfg

@@ -0,0 +1,3 @@
+from lit.llvm import llvm_config
+
+config.substitutions.append(("%llvm-driver", "1" if "llvm-driver" in config.available_features else "0"))

clang/test/Driver/DTLTO/ps5-dtlto.c

@@ -3,16 +3,18 @@
 /// Check DTLTO options are forwarded to the linker.
 
 /// Check that options are forwarded as expected with --thinlto-distributor=.
-// RUN: %python %S/filename.py %clang > %t_forward.log
+// RUN: %python %S/dtlto-helper.py %clang %llvm-driver > %t_forward.log
 // RUN: %clang -flto=thin %s -### --target=x86_64-sie-ps5 \
 // RUN:   -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 \
 // RUN:   -fthinlto-distributor=d.exe -Werror >>%t_forward.log 2>&1
 // RUN: FileCheck %s --input-file=%t_forward.log --check-prefix=FORWARD
 
-// FORWARD: filename.py:[[CLANG:.*]]
-// FORWARD: prospero-lld
+// FORWARD:      clang-name:[[CLANG:.*]]
+// FORWARD-NEXT: prepend-arg:[[PREPEND_ARG:.*]]
+// FORWARD:      prospero-lld
 // FORWARD-SAME: "--thinlto-distributor=d.exe"
 // FORWARD-SAME: "--thinlto-remote-compiler={{[^"]*}}[[CLANG]]"
+// FORWARD-SAME: [[PREPEND_ARG]]
 // FORWARD-SAME: "--thinlto-distributor-arg=a1"
 // FORWARD-SAME: "--thinlto-distributor-arg=a2"
 // FORWARD-SAME: "--thinlto-distributor-arg=a3"
@@ -30,20 +32,22 @@
 
 /// Check the expected arguments are forwarded by default with only
 /// --thinlto-distributor=.
-// RUN: %python %S/filename.py %clang > %t_default.log
+// RUN: %python %S/dtlto-helper.py %clang %llvm-driver > %t_default.log
 // RUN: %clang -flto=thin %s -### --target=x86_64-sie-ps5 \
 // RUN:   -fthinlto-distributor=d.exe -Werror >>%t_default.log 2>&1
 // RUN: FileCheck %s --input-file=%t_default.log --check-prefix=DEFAULT \
 // RUN:   --implicit-check-not=distributor --implicit-check-not=remote-compiler
 
-// DEFAULT: filename.py:[[CLANG:.*]]
-// DEFAULT: prospero-lld
+// DEFAULT:      clang-name:[[CLANG:.*]]
+// DEFAULT-NEXT: prepend-arg:[[PREPEND_ARG:.*]]
+// DEFAULT:      prospero-lld
 // DEFAULT-SAME: "--thinlto-distributor=d.exe"
 // DEFAULT-SAME: "--thinlto-remote-compiler={{[^"]*}}[[CLANG]]"
+// DEFAULT-SAME: [[PREPEND_ARG]]
 
 /// Check that the arguments are forwarded unconditionally even when the
 /// compiler is not in LTO mode.
-// RUN: %python %S/filename.py %clang > %t_noflto.log
+// RUN: %python %S/dtlto-helper.py %clang %llvm-driver > %t_noflto.log
 // RUN: %clang %s -### --target=x86_64-sie-ps5 \
 // RUN:   -fthinlto-distributor=d.exe -Werror >>%t_noflto.log 2>&1
 // RUN: FileCheck %s --input-file=%t_noflto.log --check-prefix=DEFAULT \

lld/COFF/Config.h

@@ -201,6 +201,9 @@ struct Configuration {
   // Used for /thinlto-remote-compiler:<path>
   StringRef dtltoCompiler;
 
+  // Used for /thinlto-remote-compiler-prepend-arg:<arg>
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerPrependArgs;
+
   // Used for /thinlto-remote-compiler-arg:<arg>
   llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
 

lld/COFF/Driver.cpp

@@ -2113,6 +2113,10 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     Err(ctx) << "A value must be specified for /thinlto-remote-compiler if "
                 "/thinlto-distributor is specified.";
 
+  // Handle /thinlto-remote-compiler-prepend-arg:<arg>
+  config->dtltoCompilerPrependArgs =
+      args::getStrings(args, OPT_thinlto_remote_compiler_prepend_arg);
+
   // Handle /thinlto-remote-compiler-arg:<arg>
   config->dtltoCompilerArgs =
       args::getStrings(args, OPT_thinlto_remote_compiler_arg);

lld/COFF/LTO.cpp

@@ -118,8 +118,8 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) {
         /*ShouldEmitIndexFiles=*/false,
         /*ShouldEmitImportFiles=*/false, ctx.config.outputFile,
         ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs,
-        ctx.config.dtltoCompiler, ctx.config.dtltoCompilerArgs,
-        !ctx.config.saveTempsArgs.empty());
+        ctx.config.dtltoCompiler, ctx.config.dtltoCompilerPrependArgs,
+        ctx.config.dtltoCompilerArgs, !ctx.config.saveTempsArgs.empty());
   } else if (ctx.config.thinLTOIndexOnly) {
     auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); };
     backend = lto::createWriteIndexesThinBackend(

lld/COFF/Options.td

@@ -292,6 +292,9 @@ def thinlto_distributor_arg : P<"thinlto-distributor-arg",
 def thinlto_remote_compiler : P<"thinlto-remote-compiler",
   "Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
   "compilations">;
+def thinlto_remote_compiler_prepend_arg : P<"thinlto-remote-compiler-prepend-arg",
+  "Compiler prepend arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 def thinlto_remote_compiler_arg : P<"thinlto-remote-compiler-arg",
   "Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend "
   "compilations">;

lld/ELF/Config.h

@@ -278,6 +278,7 @@ struct Config {
   llvm::StringRef dtltoDistributor;
   llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
   llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerPrependArgs;
   llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;

lld/ELF/Driver.cpp

@@ -1400,6 +1400,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   ctx.arg.dtltoDistributorArgs =
       args::getStrings(args, OPT_thinlto_distributor_arg);
   ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_remote_compiler_eq);
+  ctx.arg.dtltoCompilerPrependArgs =
+      args::getStrings(args, OPT_thinlto_remote_compiler_prepend_arg);
   ctx.arg.dtltoCompilerArgs =
       args::getStrings(args, OPT_thinlto_remote_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);

lld/ELF/LTO.cpp

@@ -186,7 +186,8 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
         ctx.arg.outputFile, ctx.arg.dtltoDistributor,
         ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
-        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
+        ctx.arg.dtltoCompilerPrependArgs, ctx.arg.dtltoCompilerArgs,
+        !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),

lld/ELF/Options.td

@@ -725,6 +725,9 @@ defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
 def thinlto_remote_compiler_eq: JJ<"thinlto-remote-compiler=">,
   HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
   "compilations">;
+defm thinlto_remote_compiler_prepend_arg: EEq<"thinlto-remote-compiler-prepend-arg",
+  "Compiler prepend arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm thinlto_remote_compiler_arg: EEq<"thinlto-remote-compiler-arg",
   "Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend "
   "compilations">;

lld/docs/DTLTO.rst

@@ -14,19 +14,23 @@ ELF LLD
 
 The command-line interface is as follows:
 
-- ``--thinlto-distributor=<path>``  
+- ``--thinlto-distributor=<path>``
   Specifies the file to execute as the distributor process. If specified,
   ThinLTO backend compilations will be distributed.
 
-- ``--thinlto-remote-compiler=<path>``  
+- ``--thinlto-remote-compiler=<path>``
   Specifies the path to the compiler that the distributor process will use for
   backend compilations. The compiler invoked must match the version of LLD.
 
-- ``--thinlto-distributor-arg=<arg>``  
+- ``--thinlto-distributor-arg=<arg>``
   Specifies ``<arg>`` on the command line when invoking the distributor.
   Can be specified multiple times.
 
-- ``--thinlto-remote-compiler-arg=<arg>``  
+- ``--thinlto-remote-compiler-prepend-arg=<arg>``
+  Prepends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``
   Appends ``<arg>`` to the remote compiler's command line.
   Can be specified multiple times.
 
@@ -57,6 +61,10 @@ The command-line interface is as follows:
   Specifies ``<arg>`` on the command line when invoking the distributor.
   Can be specified multiple times.
 
+- ``/thinlto-remote-compiler-prepend-arg:<arg>``
+  Prepends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
 - ``/thinlto-remote-compiler-arg:<arg>``
   Appends ``<arg>`` to the remote compiler's command line.
   Can be specified multiple times.
@@ -72,4 +80,4 @@ Currently, other options are silently accepted but do not have the intended
 effect. Support for such options could be expanded in the future.
 
 Currently, there is no DTLTO command line interface supplied for ``clang-cl``,
-as users are expected to invoke LLD directly.
+as users are expected to invoke LLD directly.

llvm/include/llvm/LTO/LTO.h

@@ -323,6 +323,8 @@ LLVM_ABI ThinBackend createInProcessThinBackend(
 /// distributor.
 /// RemoteCompiler specifies the path to a Clang executable to be invoked for
 /// the backend jobs.
+/// RemoteCompilerPrependArgs specifies a list of prepend arguments to be
+/// applied to the backend compilations.
 /// RemoteCompilerArgs specifies a list of arguments to be applied to the
 /// backend compilations.
 /// SaveTemps is a debugging tool that prevents temporary files created by this
@@ -332,6 +334,7 @@ LLVM_ABI ThinBackend createOutOfProcessThinBackend(
     bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
     StringRef LinkerOutputFile, StringRef Distributor,
     ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,
+    ArrayRef<StringRef> RemoteCompilerPrependArgs,
     ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps);
 
 /// This ThinBackend writes individual module indexes to files, instead of

llvm/lib/LTO/LTO.cpp

@@ -2252,6 +2252,7 @@ class OutOfProcessThinBackend : public CGThinBackend {
   ArrayRef<StringRef> DistributorArgs;
 
   SString RemoteCompiler;
+  ArrayRef<StringRef> RemoteCompilerPrependArgs;
   ArrayRef<StringRef> RemoteCompilerArgs;
 
   bool SaveTemps;
@@ -2288,12 +2289,14 @@ class OutOfProcessThinBackend : public CGThinBackend {
       bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
       StringRef LinkerOutputFile, StringRef Distributor,
       ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,
+      ArrayRef<StringRef> RemoteCompilerPrependArgs,
       ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps)
       : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
                       AddStream, OnWrite, ShouldEmitIndexFiles,
                       ShouldEmitImportsFiles, ThinLTOParallelism),
         LinkerOutputFile(LinkerOutputFile), DistributorPath(Distributor),
         DistributorArgs(DistributorArgs), RemoteCompiler(RemoteCompiler),
+        RemoteCompilerPrependArgs(RemoteCompilerPrependArgs),
         RemoteCompilerArgs(RemoteCompilerArgs), SaveTemps(SaveTemps) {}
 
   virtual void setup(unsigned ThinLTONumTasks, unsigned ThinLTOTaskOffset,
@@ -2415,6 +2418,11 @@ class OutOfProcessThinBackend : public CGThinBackend {
         JOS.attributeArray("args", [&]() {
           JOS.value(RemoteCompiler);
 
+          // Forward any supplied prepend options.
+          if (!RemoteCompilerPrependArgs.empty())
+            for (auto &A : RemoteCompilerPrependArgs)
+              JOS.value(A);
+
           JOS.value("-c");
 
           JOS.value(Saver.save("--target=" + Triple.str()));
@@ -2545,6 +2553,7 @@ ThinBackend lto::createOutOfProcessThinBackend(
     bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
     StringRef LinkerOutputFile, StringRef Distributor,
     ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,
+    ArrayRef<StringRef> RemoteCompilerPrependArgs,
     ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps) {
   auto Func =
       [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
@@ -2554,7 +2563,7 @@ ThinBackend lto::createOutOfProcessThinBackend(
             Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
             AddStream, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles,
             LinkerOutputFile, Distributor, DistributorArgs, RemoteCompiler,
-            RemoteCompilerArgs, SaveTemps);
+            RemoteCompilerPrependArgs, RemoteCompilerArgs, SaveTemps);
       };
   return ThinBackend(Func, Parallelism);
 }