Support writing to a global swift index store

"-global-index-store-path" doesn't change where the index store is output to on its own, it's only used if at that path a file named "rules_swift_global_index_enabled" exists, which needs to be managed by an external process.

Always passing this flag allows caching of the results if they are output to the non-global location. This allows for incremental compilation to use the "rules_swift_global_index_enabled" file and merge cached a fresh results (outside of Bazel).

Author: brentleyjones

swift/internal/compiling.bzl

@@ -1172,6 +1172,20 @@ def _index_while_building_configurator(prerequisites, args):
     if not _index_store_path_overridden(prerequisites.user_compile_flags):
         args.add("-index-store-path", prerequisites.indexstore_directory.path)
 
+        # "-global-index-store-path" doesn't change where the index store is
+        # output to on its own, it's only used if at that path a file named
+        # "rules_swift_global_index_enabled" exists, which needs to be managed
+        # by an external process.
+        # Always passing this flag allows caching of the results if they are
+        # output to the non-global location. This allows for incremental
+        # compilation to use the "rules_swift_global_index_enabled" file and
+        # merge cached a fresh results (outside of Bazel).
+        if prerequisites.bin_dir:
+            args.add(
+                "-Xwrapped-swift=-global-index-store-path",
+                paths.join(prerequisites.bin_dir.path, "_indexstore"),
+            )
+
 def _conditional_compilation_flag_configurator(prerequisites, args):
     """Adds (non-Clang) conditional compilation flags to the command line."""
     all_defines = depset(

tools/common/file_system.cc

@@ -37,6 +37,10 @@ std::string GetCurrentDirectory() {
   return cwd;
 }
 
+bool FileExists(const std::string &path) {
+  return access(path.c_str(), 0) == 0;
+}
+
 bool CopyFile(const std::string &src, const std::string &dest) {
 #ifdef __APPLE__
   // The `copyfile` function with `COPYFILE_ALL` mode preserves permissions and

tools/common/file_system.h

@@ -20,6 +20,9 @@
 // Gets the path to the current working directory.
 std::string GetCurrentDirectory();
 
+// Returns true if something exists at path.
+bool FileExists(const std::string &path);
+
 // Copies the file at src to dest. Returns true if successful.
 bool CopyFile(const std::string &src, const std::string &dest);
 

tools/common/string_utils.cc

@@ -40,3 +40,41 @@ bool MakeSubstitutions(std::string *arg,
 
   return changed;
 }
+
+std::string Unescape(const std::string &arg) {
+  std::string result;
+  auto length = arg.size();
+  for (size_t i = 0; i < length; ++i) {
+    auto ch = arg[i];
+
+    // If it's a backslash, consume it and append the character that follows.
+    if (ch == '\\' && i + 1 < length) {
+      ++i;
+      result.push_back(arg[i]);
+      continue;
+    }
+
+    // If it's a quote, process everything up to the matching quote, unescaping
+    // backslashed characters as needed.
+    if (ch == '"' || ch == '\'') {
+      auto quote = ch;
+      ++i;
+      while (i != length && arg[i] != quote) {
+        if (arg[i] == '\\' && i + 1 < length) {
+          ++i;
+        }
+        result.push_back(arg[i]);
+        ++i;
+      }
+      if (i == length) {
+        break;
+      }
+      continue;
+    }
+
+    // It's a regular character.
+    result.push_back(ch);
+  }
+
+  return result;
+}

tools/common/string_utils.h

@@ -22,4 +22,7 @@
 bool MakeSubstitutions(std::string *arg,
                        const std::map<std::string, std::string> &mappings);
 
+// Unescape and unquote an argument read from a line of a response file.
+std::string Unescape(const std::string &arg);
+
 #endif  // BUILD_BAZEL_RULES_SWIFT_TOOLS_COMMON_STRING_UTILS_H_

tools/worker/BUILD

@@ -46,6 +46,9 @@ cc_library(
     hdrs = ["compile_without_worker.h"],
     deps = [
         ":swift_runner",
+        "//tools/common:file_system",
+        "//tools/common:string_utils",
+        "//tools/common:temp_file",
     ],
 )
 

tools/worker/compile_without_worker.cc

@@ -14,12 +14,110 @@
 
 #include "tools/worker/compile_without_worker.h"
 
-#include <iostream>
+#include <fstream>
 #include <string>
 #include <vector>
 
+#include "tools/common/file_system.h"
+#include "tools/common/string_utils.h"
+#include "tools/common/temp_file.h"
 #include "tools/worker/swift_runner.h"
 
+namespace {
+
+static void ProcessPossibleResponseFile(
+    const std::string &arg, std::function<void(const std::string &)> consumer) {
+  auto path = arg.substr(1);
+  std::ifstream original_file(path);
+  // If we couldn't open it, maybe it's not a file; maybe it's just some other
+  // argument that starts with "@". (Unlikely, but it's safer to check.)
+  if (!original_file.good()) {
+    consumer(arg);
+    return;
+  }
+
+  std::string arg_from_file;
+  while (std::getline(original_file, arg_from_file)) {
+    // Arguments in response files might be quoted/escaped, so we need to
+    // unescape them ourselves.
+    consumer(Unescape(arg_from_file));
+  }
+}
+
+static std::tuple<bool, std::vector<std::string>>
+    ArgumentsIncludingParamsContent(const std::vector<std::string> &args) {
+  std::vector<std::string> full_args;
+  std::string prev_arg;
+  std::string index_store_path;
+  std::string global_index_store_path;
+  bool process_args = false;
+
+  auto consumer = [&](const std::string &arg) {
+    if (arg == "-index-store-path") {
+      // Peel off the `-index-store-path` argument, so we can rewrite it if
+      // necessary later.
+    } else if (prev_arg == "-index-store-path") {
+      index_store_path = arg;
+    } else if (arg == "-Xwrapped-swift=-global-index-store-path") {
+      // Minimally we want to not pass this argument to the worker, since it
+      // can't process it
+      process_args = true;
+    } else if (prev_arg == "-Xwrapped-swift=-global-index-store-path") {
+      global_index_store_path = arg;
+    } else {
+      full_args.push_back(arg);
+    }
+
+    prev_arg = arg;
+  };
+
+  for (auto arg : args) {
+    if (arg[0] == '@') {
+      ProcessPossibleResponseFile(arg, consumer);
+    } else {
+      consumer(arg);
+    }
+  }
+
+  if (!global_index_store_path.empty() && FileExists(global_index_store_path + "/rules_swift_global_index_enabled")) {
+    full_args.push_back("-index-store-path");
+    full_args.push_back(global_index_store_path);
+  } else if (!index_store_path.empty()) {
+    // Add back index store path
+    full_args.push_back("-index-store-path");
+    full_args.push_back(index_store_path);
+  }
+
+  return std::make_tuple(process_args, full_args);
+}
+
+}  // end namespace
+
 int CompileWithoutWorker(const std::vector<std::string> &args) {
-  return SwiftRunner(args).Run(&std::cerr, /*stdout_to_stderr=*/false);
+  std::vector<std::string> actual_args(args.begin() + 1, args.end());
+  auto arguments = ArgumentsIncludingParamsContent(actual_args);
+
+  bool process_args = std::get<0>(arguments);
+  if (!process_args) {
+    return SwiftRunner(args).Run(&std::cerr, /*stdout_to_stderr=*/false);
+  }
+
+  // We have to rewrite the arguments. This means that if we just try to pass
+  // the new arguments verbatim to swiftc, we might end up with a command line
+  // that's too long. Rather than try to figure out these limits (which is very
+  // OS-specific and easy to get wrong), we unconditionally write the processed
+  // arguments out to a params file.
+  auto params_file = TempFile::Create("swiftc_params.XXXXXX");
+  std::ofstream params_file_stream(params_file->GetPath());
+
+  // Write out all processed arguments
+  for (auto arg : std::get<1>(arguments)) {
+    params_file_stream << arg << '\n';
+  }
+
+  std::vector<std::string> new_args(args.begin(), args.begin() + 1);
+  new_args.push_back("@" + params_file->GetPath());
+  params_file_stream.close();
+
+  return SwiftRunner(new_args).Run(&std::cerr, /*stdout_to_stderr=*/false);
 }

tools/worker/swift_runner.cc

@@ -60,45 +60,6 @@ static std::unique_ptr<TempFile> WriteResponseFile(
   return response_file;
 }
 
-// Unescape and unquote an argument read from a line of a response file.
-static std::string Unescape(const std::string &arg) {
-  std::string result;
-  auto length = arg.size();
-  for (size_t i = 0; i < length; ++i) {
-    auto ch = arg[i];
-
-    // If it's a backslash, consume it and append the character that follows.
-    if (ch == '\\' && i + 1 < length) {
-      ++i;
-      result.push_back(arg[i]);
-      continue;
-    }
-
-    // If it's a quote, process everything up to the matching quote, unescaping
-    // backslashed characters as needed.
-    if (ch == '"' || ch == '\'') {
-      auto quote = ch;
-      ++i;
-      while (i != length && arg[i] != quote) {
-        if (arg[i] == '\\' && i + 1 < length) {
-          ++i;
-        }
-        result.push_back(arg[i]);
-        ++i;
-      }
-      if (i == length) {
-        break;
-      }
-      continue;
-    }
-
-    // It's a regular character.
-    result.push_back(ch);
-  }
-
-  return result;
-}
-
 }  // namespace
 
 SwiftRunner::SwiftRunner(const std::vector<std::string> &args,

tools/worker/work_processor.cc

@@ -61,6 +61,8 @@ void WorkProcessor::ProcessWorkRequest(
 
   OutputFileMap output_file_map;
   std::string output_file_map_path;
+  std::string index_store_path;
+  std::string global_index_store_path;
   bool is_wmo = false;
 
   std::string prev_arg;
@@ -73,6 +75,18 @@ void WorkProcessor::ProcessWorkRequest(
     } else if (prev_arg == "-output-file-map") {
       output_file_map_path = arg;
       arg.clear();
+    } else if (arg == "-index-store-path") {
+      // Peel off the `-index-store-path` argument, so we can rewrite it if
+      // necessary later.
+      arg.clear();
+    } else if (prev_arg == "-index-store-path") {
+      index_store_path = arg;
+      arg.clear();
+    } else if (arg == "-Xwrapped-swift=-global-index-store-path") {
+      arg.clear();
+    } else if (prev_arg == "-Xwrapped-swift=-global-index-store-path") {
+      global_index_store_path = arg;
+      arg.clear();
     } else if (ArgumentEnablesWMO(arg)) {
       is_wmo = true;
     }
@@ -109,6 +123,18 @@ void WorkProcessor::ProcessWorkRequest(
     }
   }
 
+  if (!index_store_path.empty()) {
+    if (!global_index_store_path.empty() && FileExists(global_index_store_path + "/rules_swift_global_index_enabled")) {
+      // Use the global index store path instead.
+      params_file_stream << "-index-store-path\n";
+      params_file_stream << global_index_store_path << '\n';
+    } else {
+      // Just put the original index store path back.
+      params_file_stream << "-index-store-path\n";
+      params_file_stream << index_store_path << '\n';
+    }
+  }
+
   processed_args.push_back("@" + params_file->GetPath());
   params_file_stream.close();