Auto merge of rust-lang#53245 - michaelwoerister:thinlto-rust-llvm, r…

…=alexcrichton

[experimental]: Build LLVM with ThinLTO enabled (2nd attempt)

This is rust-lang#51207 revived. This time, I'd like to run actual performance tests to see if it improves compile times.

config.toml.example

@@ -21,6 +21,12 @@
 # Indicates whether the LLVM build is a Release or Debug build
 #optimize = true
 
+# Indicates whether LLVM should be built with ThinLTO. Note that this will
+# only succeed if you use clang, lld, llvm-ar, and llvm-ranlib in your C/C++
+# toolchain (see the `cc`, `cxx`, `linker`, `ar`, and `ranlib` options below).
+# More info at: https://clang.llvm.org/docs/ThinLTO.html#clang-bootstrap
+#thin-lto = false
+
 # Indicates whether an LLVM Release build should include debug info
 #release-debuginfo = false
 
@@ -388,6 +394,10 @@
 # Note: an absolute path should be used, otherwise LLVM build will break.
 #ar = "ar"
 
+# Ranlib to be used to assemble static libraries compiled from C/C++ code.
+# Note: an absolute path should be used, otherwise LLVM build will break.
+#ranlib = "ranlib"
+
 # Linker to be used to link Rust code. Note that the
 # default value is platform specific, and if not specified it may also depend on
 # what platform is crossing to what platform.

src/bootstrap/builder.rs

@@ -827,7 +827,7 @@ impl<'a> Builder<'a> {
         if let Some(ref error_format) = self.config.rustc_error_format {
             cargo.env("RUSTC_ERROR_FORMAT", error_format);
         }
-        if cmd != "build" && cmd != "check" && want_rustdoc {
+        if cmd != "build" && cmd != "check" && cmd != "rustc" && want_rustdoc {
             cargo.env("RUSTDOC_LIBDIR", self.sysroot_libdir(compiler, self.config.build));
         }
 
@@ -988,7 +988,7 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if cmd == "build"
+        if (cmd == "build" || cmd == "rustc")
             && mode == Mode::Std
             && self.config.extended
             && compiler.is_final_stage(self)

src/bootstrap/check.rs

@@ -50,6 +50,7 @@ impl Step for Std {
         println!("Checking std artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   true);
 
@@ -98,6 +99,7 @@ impl Step for Rustc {
         println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   true);
 
@@ -149,6 +151,7 @@ impl Step for CodegenBackend {
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &codegen_backend_stamp(builder, compiler, target, backend),
                   true);
     }
@@ -187,6 +190,7 @@ impl Step for Test {
         println!("Checking test artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   true);
 
@@ -236,6 +240,7 @@ impl Step for Rustdoc {
         println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &rustdoc_stamp(builder, compiler, target),
                   true);
 

src/bootstrap/compile.rs

@@ -117,6 +117,7 @@ impl Step for Std {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   false);
 
@@ -396,6 +397,7 @@ impl Step for Test {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   false);
 
@@ -529,6 +531,7 @@ impl Step for Rustc {
                  compiler.stage, &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   false);
 
@@ -673,18 +676,47 @@ impl Step for CodegenBackend {
         let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
         builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
 
-        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
+        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
         cargo.arg("--manifest-path")
             .arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo);
 
         let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
 
+        let mut cargo_tails_args = vec![];
+
+        if builder.config.llvm_thin_lto {
+            cargo_tails_args.push("--".to_string());
+
+            let num_jobs = builder.jobs();
+
+            if !target.contains("msvc") {
+                // Here we assume that the linker is clang. If it's not, there'll
+                // be linker errors.
+                cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
+                cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
+
+                if builder.config.llvm_optimize {
+                    cargo_tails_args.push("-Clink-arg=-O2".to_string());
+                }
+
+                // Let's make LLD respect the `-j` option.
+                let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            } else {
+                // Here we assume that the linker is lld-link.exe. lld-link.exe
+                // does not need the extra arguments except for num_jobs
+                let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            }
+        }
+
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         let files = run_cargo(builder,
                               cargo.arg("--features").arg(features),
+                              cargo_tails_args,
                               &tmp_stamp,
                               false);
         if builder.config.dry_run {
@@ -1045,7 +1077,11 @@ fn stderr_isatty() -> bool {
     }
 }
 
-pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check: bool)
+pub fn run_cargo(builder: &Builder,
+                 cargo: &mut Command,
+                 tail_args: Vec<String>,
+                 stamp: &Path,
+                 is_check: bool)
     -> Vec<PathBuf>
 {
     if builder.config.dry_run {
@@ -1066,7 +1102,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
     // files we need to probe for later.
     let mut deps = Vec::new();
     let mut toplevel = Vec::new();
-    let ok = stream_cargo(builder, cargo, &mut |msg| {
+    let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
         let filenames = match msg {
             CargoMessage::CompilerArtifact { filenames, .. } => filenames,
             _ => return,
@@ -1191,6 +1227,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
 pub fn stream_cargo(
     builder: &Builder,
     cargo: &mut Command,
+    tail_args: Vec<String>,
     cb: &mut dyn FnMut(CargoMessage),
 ) -> bool {
     if builder.config.dry_run {
@@ -1210,6 +1247,10 @@ pub fn stream_cargo(
         cargo.env("RUSTC_COLOR", "1");
     }
 
+    for arg in tail_args {
+        cargo.arg(arg);
+    }
+
     builder.verbose(&format!("running: {:?}", cargo));
     let mut child = match cargo.spawn() {
         Ok(child) => child,

src/bootstrap/config.rs

@@ -77,6 +77,7 @@ pub struct Config {
     pub llvm_enabled: bool,
     pub llvm_assertions: bool,
     pub llvm_optimize: bool,
+    pub llvm_thin_lto: bool,
     pub llvm_release_debuginfo: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
@@ -163,6 +164,7 @@ pub struct Target {
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
+    pub ranlib: Option<PathBuf>,
     pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
@@ -246,6 +248,7 @@ struct Llvm {
     ninja: Option<bool>,
     assertions: Option<bool>,
     optimize: Option<bool>,
+    thin_lto: Option<bool>,
     release_debuginfo: Option<bool>,
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
@@ -327,6 +330,7 @@ struct TomlTarget {
     cc: Option<String>,
     cxx: Option<String>,
     ar: Option<String>,
+    ranlib: Option<String>,
     linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
@@ -503,6 +507,7 @@ impl Config {
             set(&mut config.llvm_enabled, llvm.enabled);
             llvm_assertions = llvm.assertions;
             set(&mut config.llvm_optimize, llvm.optimize);
+            set(&mut config.llvm_thin_lto, llvm.thin_lto);
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
@@ -581,6 +586,7 @@ impl Config {
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.ranlib = cfg.ranlib.clone().map(PathBuf::from);
                 target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);

src/bootstrap/dist.rs

@@ -1885,6 +1885,34 @@ impl Step for HashSign {
     }
 }
 
+// Maybe add libLLVM.so to the lib-dir. It will only have been built if
+// LLVM tools are linked dynamically.
+// Note: This function does no yet support Windows but we also don't support
+//       linking LLVM tools dynamically on Windows yet.
+fn maybe_install_llvm_dylib(builder: &Builder,
+                            target: Interned<String>,
+                            image: &Path) {
+    let src_libdir = builder
+        .llvm_out(target)
+        .join("lib");
+
+    // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
+    // Since tools link to the latter rather than the former, we have to
+    // follow the symlink to find out what to distribute.
+    let llvm_dylib_path = src_libdir.join("libLLVM.so");
+    if llvm_dylib_path.exists() {
+        let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
+            panic!("dist: Error calling canonicalize path `{}`: {}",
+                   llvm_dylib_path.display(), e);
+        });
+
+        let dst_libdir = image.join("lib");
+        t!(fs::create_dir_all(&dst_libdir));
+
+        builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
+    }
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct LlvmTools {
     pub stage: u32,
@@ -1929,18 +1957,18 @@ impl Step for LlvmTools {
         drop(fs::remove_dir_all(&image));
 
         // Prepare the image directory
-        let bindir = builder
+        let src_bindir = builder
             .llvm_out(target)
             .join("bin");
-        let dst = image.join("lib/rustlib")
-            .join(target)
-            .join("bin");
-        t!(fs::create_dir_all(&dst));
+        let dst_bindir = image.join("bin");
+        t!(fs::create_dir_all(&dst_bindir));
         for tool in LLVM_TOOLS {
-            let exe = bindir.join(exe(tool, &target));
-            builder.install(&exe, &dst, 0o755);
+            let exe = src_bindir.join(exe(tool, &target));
+            builder.install(&exe, &dst_bindir, 0o755);
         }
 
+        maybe_install_llvm_dylib(builder, target, &image);
+
         // Prepare the overlay
         let overlay = tmp.join("llvm-tools-overlay");
         drop(fs::remove_dir_all(&overlay));
@@ -2025,7 +2053,6 @@ impl Step for Lldb {
         let dst = image.join("lib");
         t!(fs::create_dir_all(&dst));
         for entry in t!(fs::read_dir(&libdir)) {
-            // let entry = t!(entry);
             let entry = entry.unwrap();
             if let Ok(name) = entry.file_name().into_string() {
                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
@@ -2060,6 +2087,9 @@ impl Step for Lldb {
             }
         }
 
+        // Copy libLLVM.so to the lib dir as well, if needed.
+        maybe_install_llvm_dylib(builder, target, &image);
+
         // Prepare the overlay
         let overlay = tmp.join("lldb-overlay");
         drop(fs::remove_dir_all(&overlay));

src/bootstrap/lib.rs

@@ -281,6 +281,7 @@ pub struct Build {
     cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
     ar: HashMap<Interned<String>, PathBuf>,
+    ranlib: HashMap<Interned<String>, PathBuf>,
     // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
@@ -406,6 +407,7 @@ impl Build {
             cc: HashMap::new(),
             cxx: HashMap::new(),
             ar: HashMap::new(),
+            ranlib: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -772,6 +774,11 @@ impl Build {
         self.ar.get(&target).map(|p| &**p)
     }
 
+    /// Returns the path to the `ranlib` utility for the target specified.
+    fn ranlib(&self, target: Interned<String>) -> Option<&Path> {
+        self.ranlib.get(&target).map(|p| &**p)
+    }
+
     /// Returns the path to the C++ compiler for the target specified.
     fn cxx(&self, target: Interned<String>) -> Result<&Path, String> {
         match self.cxx.get(&target) {
@@ -1018,6 +1025,10 @@ impl Build {
         self.rust_version()
     }
 
+    fn llvm_link_tools_dynamically(&self, target: Interned<String>) -> bool {
+        (target.contains("linux-gnu") || target.contains("apple-darwin"))
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///