ship LLVM tools with the toolchain

config.toml.example

@@ -347,6 +347,10 @@
# rustc to execute.
#lld = false

# Indicates whether some LLVM tools, like llvm-objdump, will be made available in the
# sysroot.
#llvm-tools = false

# Whether to deny warnings in crates
#deny-warnings = true

src/bootstrap/builder.rs

@@ -451,6 +451,7 @@ impl<'a> Builder<'a> {
                dist::Cargo,
                dist::Rls,
                dist::Rustfmt,
                dist::LlvmTools,
                dist::Extended,
                dist::HashSign
            ),

src/bootstrap/config.rs

@@ -88,6 +88,7 @@ pub struct Config {
    pub llvm_link_jobs: Option<u32>,

    pub lld_enabled: bool,
    pub llvm_tools_enabled: bool,

    // rust codegen options
    pub rust_optimize: bool,
@@ -308,6 +309,7 @@ struct Rust {
    codegen_backends_dir: Option<String>,
    wasm_syscall: Option<bool>,
    lld: Option<bool>,
    llvm_tools: Option<bool>,
    deny_warnings: Option<bool>,
    backtrace_on_ice: Option<bool>,
}
@@ -531,6 +533,7 @@ impl Config {
            set(&mut config.test_miri, rust.test_miri);
            set(&mut config.wasm_syscall, rust.wasm_syscall);
            set(&mut config.lld_enabled, rust.lld);
            set(&mut config.llvm_tools_enabled, rust.llvm_tools);
            config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
            config.rustc_default_linker = rust.default_linker.clone();
            config.musl_root = rust.musl_root.clone().map(PathBuf::from);

src/bootstrap/configure.py

@@ -335,6 +335,7 @@ def set(key, value):
    elif option.name == 'full-tools':
        set('rust.codegen-backends', ['llvm', 'emscripten'])
        set('rust.lld', True)
        set('rust.llvm-tools', True)
        set('build.extended', True)
    elif option.name == 'option-checking':
        # this was handled above

src/bootstrap/dist.rs

@@ -26,7 +26,7 @@ use std::process::{Command, Stdio};

use build_helper::output;

use {Compiler, Mode};
use {Compiler, Mode, LLVM_TOOLS};
use channel;
use util::{libdir, is_dylib, exe};
use builder::{Builder, RunConfig, ShouldRun, Step};
@@ -43,6 +43,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
        format!("{}-{}", component, builder.rls_package_vers())
    } else if component == "rustfmt" {
        format!("{}-{}", component, builder.rustfmt_package_vers())
    } else if component == "llvm-tools" {
        format!("{}-{}", component, builder.llvm_tools_vers())
    } else {
        assert!(component.starts_with("rust"));
        format!("{}-{}", component, builder.rust_package_vers())
@@ -394,7 +396,7 @@ impl Step for Rustc {
        let compiler = self.compiler;
        let host = self.compiler.host;

        builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, compiler.host));
        builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
        let name = pkgname(builder, "rustc");
        let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
        let _ = fs::remove_dir_all(&image);
@@ -1738,6 +1740,7 @@ impl Step for HashSign {
        cmd.arg(builder.package_vers(&builder.release_num("cargo")));
        cmd.arg(builder.package_vers(&builder.release_num("rls")));
        cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
        cmd.arg(builder.llvm_tools_vers());
        cmd.arg(addr);

        builder.create_dir(&distdir(builder));
@@ -1748,3 +1751,78 @@ impl Step for HashSign {
        assert!(status.success());
    }
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct LlvmTools {
    pub stage: u32,
    pub compiler: Compiler,
    pub target: Interned<String>,
}

impl Step for LlvmTools {
    type Output = Option<PathBuf>;
    const ONLY_HOSTS: bool = true;

    fn should_run(run: ShouldRun) -> ShouldRun {
        run.path("llvm-tools")
    }

    fn make_run(run: RunConfig) {
        run.builder.ensure(LlvmTools {
            stage: run.builder.top_stage,
            compiler: run.builder.compiler(run.builder.top_stage, run.target),
            target: run.target,
        });
    }

    fn run(self, builder: &Builder) -> Option<PathBuf> {
        let compiler = self.compiler;
        let host = compiler.host;

        let stage = self.stage;
        assert!(builder.config.extended);

        builder.info(&format!("Dist LlvmTools stage{} ({})", stage, host));
        let src = builder.src.join("src/llvm");
        let name = pkgname(builder, "llvm-tools");

        let tmp = tmpdir(builder);
        let image = tmp.join("llvm-tools-image");
        drop(fs::remove_dir_all(&image));
        t!(fs::create_dir_all(&image.join("bin")));

        // Prepare the image directory
        for tool in LLVM_TOOLS {
            let exe = builder
                .llvm_out(host)
                .join("bin")
                .join(exe(tool, &compiler.host));
            builder.install(&exe, &image.join("bin"), 0o755);
        }

        // Prepare the overlay
        let overlay = tmp.join("llvm-tools-overlay");
        drop(fs::remove_dir_all(&overlay));
        builder.create_dir(&overlay);
        builder.install(&src.join("README.txt"), &overlay, 0o644);
        builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);

        // Generate the installer tarball
        let mut cmd = rust_installer(builder);
        cmd.arg("generate")
            .arg("--product-name=Rust")
            .arg("--rel-manifest-dir=rustlib")
            .arg("--success-message=llvm-tools-installed.")
            .arg("--image-dir").arg(&image)
            .arg("--work-dir").arg(&tmpdir(builder))
            .arg("--output-dir").arg(&distdir(builder))
            .arg("--non-installed-overlay").arg(&overlay)
            .arg(format!("--package-name={}-{}", name, host))
            .arg("--legacy-manifest-dirs=rustlib,cargo")
            .arg("--component-name=llvm-tools");


        builder.run(&mut cmd);
        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
    }
}

src/bootstrap/lib.rs

@@ -199,6 +199,14 @@ use flags::Subcommand;
use cache::{Interned, INTERNER};
use toolstate::ToolState;

const LLVM_TOOLS: &[&str] = &[
    "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility
    "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume
    "llvm-objdump", // used to disassemble programs
    "llvm-profdata", // used to inspect and merge files generated by profiles
    "llvm-size", // prints the size of the linker sections of a program
];

/// A structure representing a Rust compiler.
///
/// Each compiler has a `stage` that it is associated with and a `host` that
@@ -949,6 +957,27 @@ impl Build {
        self.package_vers(&self.release_num("rustfmt"))
    }

    fn llvm_tools_vers(&self) -> String {
        // japaric: should we use LLVM version here?
        // let stdout = build_helper::output(
        //     Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size"))
        //         .arg("--version"),
        // );

        // for line in stdout.lines() {
        //     if line.contains("LLVM version") {
        //         if let Some(vers) = line.split_whitespace().nth(2) {
        //             return vers.to_string();
        //         }
        //     }
        // }

        // panic!("The output of $LLVM_TOOL has changed; \
        //         please fix `bootstrap::Build.llvm_tools_vers`");

        self.rust_version()
    }

    /// Returns the `version` string associated with this compiler for Rust
    /// itself.
    ///

src/bootstrap/native.rs

@@ -167,8 +167,22 @@ impl Step for Llvm {
        // which saves both memory during parallel links and overall disk space
        // for the tools.  We don't distribute any of those tools, so this is
        // just a local concern.  However, it doesn't work well everywhere.
        if target.contains("linux-gnu") || target.contains("apple-darwin") {
           cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
        //
        // If we are shipping llvm tools then we statically link them LLVM
        if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
            !builder.config.llvm_tools_enabled {
                cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
        }

        // For distribution we want the LLVM tools to be *statically* linked to libstdc++
        if builder.config.llvm_tools_enabled {
            if !target.contains("windows") {
                if target.contains("apple") {
                    cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++");
                } else {
                    cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++");
                }
            }
        }

        if target.contains("msvc") {