[Support] Class for response file expansion (NFC)

Functions that implement expansion of response and config files depend
on many options, which are passes as arguments. Extending the expansion
requires new options, it in turn causes changing calls in various places
making them even more bulky.

This change introduces a class ExpansionContext, which represents set of
options that control the expansion. Its methods implements expansion of
responce files including config files. It makes extending the expansion
easier.

No functional changes.

Differential Revision: https://reviews.llvm.org/D132379

Author: spavloff

clang/lib/Driver/Driver.cpp

@@ -956,7 +956,9 @@ static void appendOneArg(InputArgList &Args, const Arg *Opt,
 bool Driver::readConfigFile(StringRef FileName) {
   // Try reading the given file.
   SmallVector<const char *, 32> NewCfgArgs;
-  if (!llvm::cl::readConfigFile(FileName, Saver, NewCfgArgs, getVFS())) {
+  llvm::cl::ExpansionContext ExpCtx(Alloc, llvm::cl::tokenizeConfigFile);
+  ExpCtx.setVFS(&getVFS());
+  if (!ExpCtx.readConfigFile(FileName, NewCfgArgs)) {
     Diag(diag::err_drv_cannot_read_config_file) << FileName;
     return true;
   }

clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp

@@ -60,9 +60,10 @@ class ExpandResponseFilesDatabase : public CompilationDatabase {
       if (!SeenRSPFile)
         continue;
       llvm::BumpPtrAllocator Alloc;
-      llvm::StringSaver Saver(Alloc);
-      llvm::cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false, false,
-                                    llvm::StringRef(Cmd.Directory), *FS);
+      llvm::cl::ExpansionContext ECtx(Alloc, Tokenizer);
+      ECtx.setVFS(FS.get())
+          .setCurrentDir(Cmd.Directory)
+          .expandResponseFiles(Argv);
       // Don't assign directly, Argv aliases CommandLine.
       std::vector<std::string> ExpandedArgv(Argv.begin(), Argv.end());
       Cmd.CommandLine = std::move(ExpandedArgv);

clang/tools/driver/driver.cpp

@@ -308,9 +308,8 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV) {
   llvm::cl::ResetAllOptionOccurrences();
 
   llvm::BumpPtrAllocator A;
-  llvm::StringSaver Saver(A);
-  llvm::cl::ExpandResponseFiles(Saver, &llvm::cl::TokenizeGNUCommandLine, ArgV,
-                                /*MarkEOLs=*/false);
+  llvm::cl::ExpansionContext ECtx(A, llvm::cl::TokenizeGNUCommandLine);
+  ECtx.expandResponseFiles(ArgV);
   StringRef Tool = ArgV[1];
   void *GetExecutablePathVP = (void *)(intptr_t)GetExecutablePath;
   if (Tool == "-cc1")
@@ -373,7 +372,8 @@ int clang_main(int Argc, char **Argv) {
 
   if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
     MarkEOLs = false;
-  llvm::cl::ExpandResponseFiles(Saver, Tokenizer, Args, MarkEOLs);
+  llvm::cl::ExpansionContext ECtx(A, Tokenizer);
+  ECtx.setMarkEOLs(MarkEOLs).expandResponseFiles(Args);
 
   // Handle -cc1 integrated tools, even if -cc1 was expanded from a response
   // file.

llvm/include/llvm/Support/CommandLine.h

@@ -31,6 +31,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <climits>
@@ -2065,56 +2066,88 @@ void tokenizeConfigFile(StringRef Source, StringSaver &Saver,
                         SmallVectorImpl<const char *> &NewArgv,
                         bool MarkEOLs = false);
 
-/// Reads command line options from the given configuration file.
-///
-/// \param [in] CfgFileName Path to configuration file.
-/// \param [in] Saver  Objects that saves allocated strings.
-/// \param [out] Argv Array to which the read options are added.
-/// \return true if the file was successfully read.
-///
-/// It reads content of the specified file, tokenizes it and expands "@file"
-/// commands resolving file names in them relative to the directory where
-/// CfgFilename resides. It also expands "<CFGDIR>" to the base path of the
-/// current config file.
-///
-bool readConfigFile(StringRef CfgFileName, StringSaver &Saver,
-                    SmallVectorImpl<const char *> &Argv,
-                    llvm::vfs::FileSystem &FS);
-
-/// Expand response files on a command line recursively using the given
-/// StringSaver and tokenization strategy.  Argv should contain the command line
-/// before expansion and will be modified in place. If requested, Argv will
-/// also be populated with nullptrs indicating where each response file line
-/// ends, which is useful for the "/link" argument that needs to consume all
-/// remaining arguments only until the next end of line, when in a response
-/// file.
-///
-/// \param [in] Saver Delegates back to the caller for saving parsed strings.
-/// \param [in] Tokenizer Tokenization strategy. Typically Unix or Windows.
-/// \param [in,out] Argv Command line into which to expand response files.
-/// \param [in] MarkEOLs Mark end of lines and the end of the response file
-/// with nullptrs in the Argv vector.
-/// \param [in] RelativeNames true if names of nested response files must be
-/// resolved relative to including file.
-/// \param [in] ExpandBasePath If true, "<CFGDIR>" expands to the base path of
-/// the current response file.
-/// \param [in] FS File system used for all file access when running the tool.
-/// \param [in] CurrentDir Path used to resolve relative rsp files. If set to
-/// None, the file system current directory is used instead.
+/// Contains options that control response file expansion.
+class ExpansionContext {
+  /// Provides persistent storage for parsed strings.
+  StringSaver Saver;
+
+  /// Tokenization strategy. Typically Unix or Windows.
+  TokenizerCallback Tokenizer;
+
+  /// File system used for all file access when running the expansion.
+  vfs::FileSystem *FS;
+
+  /// Path used to resolve relative rsp files. If empty, the file system
+  /// current directory is used instead.
+  StringRef CurrentDir;
+
+  /// True if names of nested response files must be resolved relative to
+  /// including file.
+  bool RelativeNames = false;
+
+  /// If true, mark end of lines and the end of the response file with nullptrs
+  /// in the Argv vector.
+  bool MarkEOLs = false;
+
+  /// If true, body of config file is expanded.
+  bool InConfigFile = false;
+
+  llvm::Error expandResponseFile(StringRef FName,
+                                 SmallVectorImpl<const char *> &NewArgv);
+
+public:
+  ExpansionContext(BumpPtrAllocator &A, TokenizerCallback T);
+
+  ExpansionContext &setMarkEOLs(bool X) {
+    MarkEOLs = X;
+    return *this;
+  }
+
+  ExpansionContext &setRelativeNames(bool X) {
+    RelativeNames = X;
+    return *this;
+  }
+
+  ExpansionContext &setCurrentDir(StringRef X) {
+    CurrentDir = X;
+    return *this;
+  }
+
+  ExpansionContext &setVFS(vfs::FileSystem *X) {
+    FS = X;
+    return *this;
+  }
+
+  /// Reads command line options from the given configuration file.
+  ///
+  /// \param [in] CfgFile Path to configuration file.
+  /// \param [out] Argv Array to which the read options are added.
+  /// \return true if the file was successfully read.
+  ///
+  /// It reads content of the specified file, tokenizes it and expands "@file"
+  /// commands resolving file names in them relative to the directory where
+  /// CfgFilename resides. It also expands "<CFGDIR>" to the base path of the
+  /// current config file.
+  bool readConfigFile(StringRef CfgFile, SmallVectorImpl<const char *> &Argv);
+
+  /// Expands constructs "@file" in the provided array of arguments recursively.
+  bool expandResponseFiles(SmallVectorImpl<const char *> &Argv);
+};
+
+/// A convenience helper which concatenates the options specified by the
+/// environment variable EnvVar and command line options, then expands
+/// response files recursively.
 /// \return true if all @files were expanded successfully or there were none.
-bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
-                         SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
-                         bool RelativeNames, bool ExpandBasePath,
-                         llvm::Optional<llvm::StringRef> CurrentDir,
-                         llvm::vfs::FileSystem &FS);
-
-/// An overload of ExpandResponseFiles() that uses
-/// llvm::vfs::getRealFileSystem().
-bool ExpandResponseFiles(
-    StringSaver &Saver, TokenizerCallback Tokenizer,
-    SmallVectorImpl<const char *> &Argv, bool MarkEOLs = false,
-    bool RelativeNames = false, bool ExpandBasePath = false,
-    llvm::Optional<llvm::StringRef> CurrentDir = llvm::None);
+bool expandResponseFiles(int Argc, const char *const *Argv, const char *EnvVar,
+                         SmallVectorImpl<const char *> &NewArgv);
+
+/// A convenience helper which supports the typical use case of expansion
+/// function call.
+inline bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
+                                SmallVectorImpl<const char *> &Argv) {
+  ExpansionContext ECtx(Saver.getAllocator(), Tokenizer);
+  return ECtx.expandResponseFiles(Argv);
+}
 
 /// A convenience helper which concatenates the options specified by the
 /// environment variable EnvVar and command line options, then expands response

llvm/include/llvm/Support/StringSaver.h

@@ -24,6 +24,8 @@ class StringSaver final {
 public:
   StringSaver(BumpPtrAllocator &Alloc) : Alloc(Alloc) {}
 
+  BumpPtrAllocator &getAllocator() const { return Alloc; }
+
   // All returned strings are null-terminated: *save(S).end() == 0.
   StringRef save(const char *S) { return save(StringRef(S)); }
   StringRef save(StringRef S);

llvm/lib/Support/CommandLine.cpp

@@ -1153,15 +1153,12 @@ static void ExpandBasePaths(StringRef BasePath, StringSaver &Saver,
 }
 
 // FName must be an absolute path.
-static llvm::Error ExpandResponseFile(StringRef FName, StringSaver &Saver,
-                                      TokenizerCallback Tokenizer,
-                                      SmallVectorImpl<const char *> &NewArgv,
-                                      bool MarkEOLs, bool RelativeNames,
-                                      bool ExpandBasePath,
-                                      llvm::vfs::FileSystem &FS) {
+llvm::Error
+ExpansionContext::expandResponseFile(StringRef FName,
+                                     SmallVectorImpl<const char *> &NewArgv) {
   assert(sys::path::is_absolute(FName));
   llvm::ErrorOr<std::unique_ptr<MemoryBuffer>> MemBufOrErr =
-      FS.getBufferForFile(FName);
+      FS->getBufferForFile(FName);
   if (!MemBufOrErr)
     return llvm::errorCodeToError(MemBufOrErr.getError());
   MemoryBuffer &MemBuf = *MemBufOrErr.get();
@@ -1196,7 +1193,7 @@ static llvm::Error ExpandResponseFile(StringRef FName, StringSaver &Saver,
       continue;
 
     // Substitute <CFGDIR> with the file's base path.
-    if (ExpandBasePath)
+    if (InConfigFile)
       ExpandBasePaths(BasePath, Saver, Arg);
 
     // Skip non-rsp file arguments.
@@ -1219,11 +1216,8 @@ static llvm::Error ExpandResponseFile(StringRef FName, StringSaver &Saver,
 
 /// Expand response files on a command line recursively using the given
 /// StringSaver and tokenization strategy.
-bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
-                             SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
-                             bool RelativeNames, bool ExpandBasePath,
-                             llvm::Optional<llvm::StringRef> CurrentDir,
-                             llvm::vfs::FileSystem &FS) {
+bool ExpansionContext::expandResponseFiles(
+    SmallVectorImpl<const char *> &Argv) {
   bool AllExpanded = true;
   struct ResponseFileRecord {
     std::string File;
@@ -1264,28 +1258,28 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
     // always have an absolute path deduced from the containing file.
     SmallString<128> CurrDir;
     if (llvm::sys::path::is_relative(FName)) {
-      if (!CurrentDir) {
-        if (auto CWD = FS.getCurrentWorkingDirectory()) {
+      if (CurrentDir.empty()) {
+        if (auto CWD = FS->getCurrentWorkingDirectory()) {
           CurrDir = *CWD;
         } else {
           // TODO: The error should be propagated up the stack.
           llvm::consumeError(llvm::errorCodeToError(CWD.getError()));
           return false;
         }
       } else {
-        CurrDir = *CurrentDir;
+        CurrDir = CurrentDir;
       }
       llvm::sys::path::append(CurrDir, FName);
       FName = CurrDir.c_str();
     }
-    auto IsEquivalent = [FName, &FS](const ResponseFileRecord &RFile) {
-      llvm::ErrorOr<llvm::vfs::Status> LHS = FS.status(FName);
+    auto IsEquivalent = [FName, this](const ResponseFileRecord &RFile) {
+      llvm::ErrorOr<llvm::vfs::Status> LHS = FS->status(FName);
       if (!LHS) {
         // TODO: The error should be propagated up the stack.
         llvm::consumeError(llvm::errorCodeToError(LHS.getError()));
         return false;
       }
-      llvm::ErrorOr<llvm::vfs::Status> RHS = FS.status(RFile.File);
+      llvm::ErrorOr<llvm::vfs::Status> RHS = FS->status(RFile.File);
       if (!RHS) {
         // TODO: The error should be propagated up the stack.
         llvm::consumeError(llvm::errorCodeToError(RHS.getError()));
@@ -1306,9 +1300,7 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
     // Replace this response file argument with the tokenization of its
     // contents.  Nested response files are expanded in subsequent iterations.
     SmallVector<const char *, 0> ExpandedArgv;
-    if (llvm::Error Err =
-            ExpandResponseFile(FName, Saver, Tokenizer, ExpandedArgv, MarkEOLs,
-                               RelativeNames, ExpandBasePath, FS)) {
+    if (llvm::Error Err = expandResponseFile(FName, ExpandedArgv)) {
       // We couldn't read this file, so we leave it in the argument stream and
       // move on.
       // TODO: The error should be propagated up the stack.
@@ -1338,15 +1330,6 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
   return AllExpanded;
 }
 
-bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
-                             SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
-                             bool RelativeNames, bool ExpandBasePath,
-                             llvm::Optional<StringRef> CurrentDir) {
-  return ExpandResponseFiles(Saver, std::move(Tokenizer), Argv, MarkEOLs,
-                             RelativeNames, ExpandBasePath,
-                             std::move(CurrentDir), *vfs::getRealFileSystem());
-}
-
 bool cl::expandResponseFiles(int Argc, const char *const *Argv,
                              const char *EnvVar, StringSaver &Saver,
                              SmallVectorImpl<const char *> &NewArgv) {
@@ -1360,30 +1343,30 @@ bool cl::expandResponseFiles(int Argc, const char *const *Argv,
 
   // Command line options can override the environment variable.
   NewArgv.append(Argv + 1, Argv + Argc);
-  return ExpandResponseFiles(Saver, Tokenize, NewArgv);
+  ExpansionContext ECtx(Saver.getAllocator(), Tokenize);
+  return ECtx.expandResponseFiles(NewArgv);
 }
 
-bool cl::readConfigFile(StringRef CfgFile, StringSaver &Saver,
-                        SmallVectorImpl<const char *> &Argv,
-                        llvm::vfs::FileSystem &FS) {
+ExpansionContext::ExpansionContext(BumpPtrAllocator &A, TokenizerCallback T)
+    : Saver(A), Tokenizer(T), FS(vfs::getRealFileSystem().get()) {}
+
+bool ExpansionContext::readConfigFile(StringRef CfgFile,
+                                      SmallVectorImpl<const char *> &Argv) {
   SmallString<128> AbsPath;
   if (sys::path::is_relative(CfgFile)) {
     AbsPath.assign(CfgFile);
-    if (std::error_code EC = FS.makeAbsolute(AbsPath))
+    if (std::error_code EC = FS->makeAbsolute(AbsPath))
       return false;
     CfgFile = AbsPath.str();
   }
-  if (llvm::Error Err =
-          ExpandResponseFile(CfgFile, Saver, cl::tokenizeConfigFile, Argv,
-                             /*MarkEOLs=*/false, /*RelativeNames=*/true,
-                             /*ExpandBasePath=*/true, FS)) {
+  InConfigFile = true;
+  RelativeNames = true;
+  if (llvm::Error Err = expandResponseFile(CfgFile, Argv)) {
     // TODO: The error should be propagated up the stack.
     llvm::consumeError(std::move(Err));
     return false;
   }
-  return ExpandResponseFiles(Saver, cl::tokenizeConfigFile, Argv,
-                             /*MarkEOLs=*/false, /*RelativeNames=*/true,
-                             /*ExpandBasePath=*/true, llvm::None, FS);
+  return expandResponseFiles(Argv);
 }
 
 static void initCommonOptions();
@@ -1441,11 +1424,10 @@ bool CommandLineParser::ParseCommandLineOptions(int argc,
   // Expand response files.
   SmallVector<const char *, 20> newArgv(argv, argv + argc);
   BumpPtrAllocator A;
-  StringSaver Saver(A);
-  ExpandResponseFiles(Saver,
-         Triple(sys::getProcessTriple()).isOSWindows() ?
-         cl::TokenizeWindowsCommandLine : cl::TokenizeGNUCommandLine,
-         newArgv);
+  ExpansionContext ECtx(A, Triple(sys::getProcessTriple()).isOSWindows()
+                               ? cl::TokenizeWindowsCommandLine
+                               : cl::TokenizeGNUCommandLine);
+  ECtx.expandResponseFiles(newArgv);
   argv = &newArgv[0];
   argc = static_cast<int>(newArgv.size());