[Driver] Enable nested configuration files

Users may partition parameters specified by configuration file and put
different groups into separate files. These files are inserted into the
main file using constructs `@file`. Relative file names in it are
resolved relative to the including configuration file and this is not
convenient in some cases. A configuration file, which resides in system
directory, may need to include a file with user-defined parameters and
still provide default definitions if such file is absent.

To solve such problems, the option `--config=` is allowed inside
configuration files. Like `@file` it results in insertion of
command-line arguments but the algorithm of file search is different and
allows overriding system definitions with user ones.

Differential Revision: https://reviews.llvm.org/D136354

clang/docs/UsersManual.rst

@@ -985,7 +985,10 @@ configuration file:
 Files included by ``@file`` directives in configuration files are resolved
 relative to the including file. For example, if a configuration file
 ``~/.llvm/target.cfg`` contains the directive ``@os/linux.opts``, the file
-``linux.opts`` is searched for in the directory ``~/.llvm/os``.
+``linux.opts`` is searched for in the directory ``~/.llvm/os``. Another way to
+include a file content is using the command line option ``--config=``. It works
+similarly but the included file is searched for using the rules for configuration
+files.
 
 To generate paths relative to the configuration file, the ``<CFGDIR>`` token may
 be used. This will expand to the absolute path of the directory containing the

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -225,8 +225,6 @@ def note_drv_config_file_searched_in : Note<
   "was searched for in the directory: %0">;
 def err_drv_cannot_read_config_file : Error<
   "cannot read configuration file '%0': %1">;
-def err_drv_nested_config_file: Error<
-  "option '--config' is not allowed inside configuration file">;
 def err_drv_arg_requires_bitcode_input: Error<
   "option '%0' requires input to be LLVM bitcode">;
 

clang/lib/Driver/Driver.cpp

@@ -970,11 +970,6 @@ bool Driver::readConfigFile(StringRef FileName,
   if (ContainErrors)
     return true;
 
-  if (NewOptions->hasArg(options::OPT_config)) {
-    Diag(diag::err_drv_nested_config_file);
-    return true;
-  }
-
   // Claim all arguments that come from a configuration file so that the driver
   // does not warn on any that is unused.
   for (Arg *A : *NewOptions)

clang/test/Driver/Inputs/config-6.cfg

@@ -1 +1 @@
---config config-5
+--config=config-4.cfg

clang/test/Driver/config-file-errs.c

@@ -4,12 +4,6 @@
 // CHECK-MISSING-FILE: argument to '--config' is missing (expected 1 value)
 
 
-//--- '--config' must not be found in config files.
-//
-// RUN: not %clang --config %S/Inputs/config-6.cfg 2>&1 | FileCheck %s -check-prefix CHECK-NESTED
-// CHECK-NESTED: option '--config' is not allowed inside configuration file
-
-
 //--- Argument of '--config' must be existing file, if it is specified by path.
 //
 // RUN: not %clang --config somewhere/nonexistent-config-file 2>&1 | FileCheck %s -check-prefix CHECK-NONEXISTENT

clang/test/Driver/config-file.c

@@ -44,9 +44,10 @@
 // CHECK-NESTED: Configuration file: {{.*}}Inputs{{.}}config-2.cfg
 // CHECK-NESTED: -Wundefined-func-template
 
-// RUN: %clang --config-system-dir=%S/Inputs --config-user-dir= --config config-2.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NESTED2
-// CHECK-NESTED2: Configuration file: {{.*}}Inputs{{.}}config-2.cfg
-// CHECK-NESTED2: -Wundefined-func-template
+// RUN: %clang --config-system-dir=%S/Inputs --config-user-dir=%S/Inputs/config --config config-6.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NESTED2
+// CHECK-NESTED2: Configuration file: {{.*}}Inputs{{.}}config-6.cfg
+// CHECK-NESTED2: -isysroot
+// CHECK-NESTED2-SAME: /opt/data
 
 
 // RUN: %clang --config %S/Inputs/config-2a.cfg -S %s -### 2>&1 | FileCheck %s -check-prefix CHECK-NESTEDa

clang/unittests/Driver/ToolChainTest.cpp

@@ -652,4 +652,56 @@ TEST(ToolChainTest, ConfigRecursiveInclude) {
 #undef INCLUDED1
 }
 
+TEST(ToolChainTest, NestedConfigFile) {
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  struct TestDiagnosticConsumer : public DiagnosticConsumer {};
+  DiagnosticsEngine Diags(DiagID, &*DiagOpts, new TestDiagnosticConsumer);
+  IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS(
+      new llvm::vfs::InMemoryFileSystem);
+
+#ifdef _WIN32
+  const char *TestRoot = "C:\\";
+#else
+  const char *TestRoot = "/";
+#endif
+  FS->setCurrentWorkingDirectory(TestRoot);
+
+  FS->addFile("/opt/sdk/root.cfg", 0,
+              llvm::MemoryBuffer::getMemBuffer("--config=platform.cfg\n"));
+  FS->addFile("/opt/sdk/platform.cfg", 0,
+              llvm::MemoryBuffer::getMemBuffer("--sysroot=/platform-sys\n"));
+  FS->addFile("/home/test/bin/platform.cfg", 0,
+              llvm::MemoryBuffer::getMemBuffer("--sysroot=/platform-bin\n"));
+
+  SmallString<128> ClangExecutable("/home/test/bin/clang");
+  FS->makeAbsolute(ClangExecutable);
+
+  // User file is absent - use system definitions.
+  {
+    Driver TheDriver(ClangExecutable, "arm-linux-gnueabi", Diags,
+                     "clang LLVM compiler", FS);
+    std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
+        {"/home/test/bin/clang", "--config", "root.cfg",
+         "--config-system-dir=/opt/sdk", "--config-user-dir=/home/test/sdk"}));
+    ASSERT_TRUE(C);
+    ASSERT_FALSE(C->containsError());
+    EXPECT_EQ("/platform-sys", TheDriver.SysRoot);
+  }
+
+  // User file overrides system definitions.
+  FS->addFile("/home/test/sdk/platform.cfg", 0,
+              llvm::MemoryBuffer::getMemBuffer("--sysroot=/platform-user\n"));
+  {
+    Driver TheDriver(ClangExecutable, "arm-linux-gnueabi", Diags,
+                     "clang LLVM compiler", FS);
+    std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
+        {"/home/test/bin/clang", "--config", "root.cfg",
+         "--config-system-dir=/opt/sdk", "--config-user-dir=/home/test/sdk"}));
+    ASSERT_TRUE(C);
+    ASSERT_FALSE(C->containsError());
+    EXPECT_EQ("/platform-user", TheDriver.SysRoot);
+  }
+}
+
 } // end anonymous namespace.

llvm/lib/Support/CommandLine.cpp

@@ -1191,27 +1191,44 @@ Error ExpansionContext::expandResponseFile(
     return Error::success();
 
   StringRef BasePath = llvm::sys::path::parent_path(FName);
-  for (auto I = NewArgv.begin(), E = NewArgv.end(); I != E; ++I) {
-    const char *&Arg = *I;
-    if (Arg == nullptr)
+  for (const char *&Arg : NewArgv) {
+    if (!Arg)
       continue;
 
     // Substitute <CFGDIR> with the file's base path.
     if (InConfigFile)
       ExpandBasePaths(BasePath, Saver, Arg);
 
-    // Get expanded file name.
-    StringRef FileName(Arg);
-    if (!FileName.consume_front("@"))
-      continue;
-    if (!llvm::sys::path::is_relative(FileName))
+    // Discover the case, when argument should be transformed into '@file' and
+    // evaluate 'file' for it.
+    StringRef ArgStr(Arg);
+    StringRef FileName;
+    bool ConfigInclusion = false;
+    if (ArgStr.consume_front("@")) {
+      FileName = ArgStr;
+      if (!llvm::sys::path::is_relative(FileName))
+        continue;
+    } else if (ArgStr.consume_front("--config=")) {
+      FileName = ArgStr;
+      ConfigInclusion = true;
+    } else {
       continue;
+    }
 
     // Update expansion construct.
     SmallString<128> ResponseFile;
     ResponseFile.push_back('@');
-    ResponseFile.append(BasePath);
-    llvm::sys::path::append(ResponseFile, FileName);
+    if (ConfigInclusion && !llvm::sys::path::has_parent_path(FileName)) {
+      SmallString<128> FilePath;
+      if (!findConfigFile(FileName, FilePath))
+        return createStringError(
+            std::make_error_code(std::errc::no_such_file_or_directory),
+            "cannot not find configuration file: " + FileName);
+      ResponseFile.append(FilePath);
+    } else {
+      ResponseFile.append(BasePath);
+      llvm::sys::path::append(ResponseFile, FileName);
+    }
     Arg = Saver.save(ResponseFile.str()).data();
   }
   return Error::success();