Auto merge of #48125 - alexcrichton:lld, r=Mark-Simulacrum

rust: Import LLD for linking wasm objects

This commit imports the LLD project from LLVM to serve as the default linker for
the `wasm32-unknown-unknown` target. The `binaryen` submoule is consequently
removed along with "binaryen linker" support in rustc.

Moving to LLD brings with it a number of benefits for wasm code:

* LLD is itself an actual linker, so there's no need to compile all wasm code
  with LTO any more. As a result builds should be *much* speedier as LTO is no
  longer forcibly enabled for all builds of the wasm target.
* LLD is quickly becoming an "official solution" for linking wasm code together.
  This, I believe at least, is intended to be the main supported linker for
  native code and wasm moving forward. Picking up support early on should help
  ensure that we can help LLD identify bugs and otherwise prove that it works
  great for all our use cases!
* Improvements to the wasm toolchain are currently primarily focused around LLVM
  and LLD (from what I can tell at least), so it's in general much better to be
  on this bandwagon for bugfixes and new features.
* Historical "hacks" like `wasm-gc` will soon no longer be necessary, LLD
  will [natively implement][gc] `--gc-sections` (better than `wasm-gc`!) which
  means a postprocessor is no longer needed to show off Rust's "small wasm
  binary size".

LLD is added in a pretty standard way to rustc right now. A new rustbuild target
was defined for building LLD, and this is executed when a compiler's sysroot is
being assembled. LLD is compiled against the LLVM that we've got in tree, which
means we're currently on the `release_60` branch, but this may get upgraded in
the near future!

LLD is placed into rustc's sysroot in a `bin` directory. This is similar to
where `gcc.exe` can be found on Windows. This directory is automatically added
to `PATH` whenever rustc executes the linker, allowing us to define a `WasmLd`
linker which implements the interface that `wasm-ld`, LLD's frontend, expects.

Like Emscripten the LLD target is currently only enabled for Tier 1 platforms,
notably OSX/Windows/Linux, and will need to be installed manually for compiling
to wasm on other platforms. LLD is by default turned off in rustbuild, and
requires a `config.toml` option to be enabled to turn it on.

Finally the unstable `#![wasm_import_memory]` attribute was also removed as LLD
has a native option for controlling this.

[gc]: https://reviews.llvm.org/D42511

.gitmodules

@@ -41,9 +41,6 @@
 [submodule "src/dlmalloc"]
 	path = src/dlmalloc
 	url = https://github.com/alexcrichton/dlmalloc-rs.git
-[submodule "src/binaryen"]
-	path = src/binaryen
-	url = https://github.com/alexcrichton/binaryen.git
 [submodule "src/doc/rust-by-example"]
 	path = src/doc/rust-by-example
 	url = https://github.com/rust-lang/rust-by-example
@@ -53,3 +50,6 @@
 [submodule "src/stdsimd"]
 	path = src/stdsimd
 	url = https://github.com/rust-lang-nursery/stdsimd
+[submodule "src/tools/lld"]
+	path = src/tools/lld
+	url = https://github.com/rust-lang/lld.git

.travis.yml

@@ -81,7 +81,7 @@ matrix:
     # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7.
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended --enable-profiler --enable-emscripten"
+        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -95,7 +95,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers --enable-profiler --enable-emscripten"
+        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1

README.md

@@ -129,9 +129,6 @@ CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.
 python x.py build
 ```
 
-If you are seeing build failure when compiling `rustc_binaryen`, make sure the path
-length of the rust folder is not longer than 22 characters.
-
 #### Specifying an ABI
 [specifying-an-abi]: #specifying-an-abi
 

appveyor.yml

@@ -67,29 +67,27 @@ environment:
   # 32/64 bit MSVC and GNU deployment
   - RUST_CONFIGURE_ARGS: >
       --build=x86_64-pc-windows-msvc
-      --enable-extended
+      --enable-full-tools
       --enable-profiler
-      --enable-emscripten
     SCRIPT: python x.py dist
     DEPLOY: 1
   - RUST_CONFIGURE_ARGS: >
       --build=i686-pc-windows-msvc
       --target=i586-pc-windows-msvc
-      --enable-extended
+      --enable-full-tools
       --enable-profiler
-      --enable-emscripten
     SCRIPT: python x.py dist
     DEPLOY: 1
   - MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-extended --enable-emscripten
+    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools
     SCRIPT: python x.py dist
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
     DEPLOY: 1
   - MSYS_BITS: 64
     SCRIPT: python x.py dist
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-extended --enable-emscripten
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
     MINGW_DIR: mingw64

config.toml.example

@@ -329,6 +329,10 @@
 # target, as without this option the test output will not be captured.
 #wasm-syscall = false
 
+# Indicates whether LLD will be compiled and made available in the sysroot for
+# rustc to execute.
+#lld = false
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/bin/rustc.rs

@@ -224,7 +224,7 @@ fn main() {
                 // flesh out rpath support more fully in the future.
                 cmd.arg("-Z").arg("osx-rpath-install-name");
                 Some("-Wl,-rpath,@loader_path/../lib")
-            } else if !target.contains("windows") {
+            } else if !target.contains("windows") && !target.contains("wasm32") {
                 Some("-Wl,-rpath,$ORIGIN/../lib")
             } else {
                 None

src/bootstrap/bootstrap.py

@@ -641,6 +641,10 @@ def update_submodules(self):
                     continue
                 if self.get_toml('jemalloc'):
                     continue
+            if module.endswith("lld"):
+                config = self.get_toml('lld')
+                if config is None or config == 'false':
+                    continue
             filtered_submodules.append(module)
         run(["git", "submodule", "update",
              "--init", "--recursive"] + filtered_submodules,

src/bootstrap/builder.rs

@@ -316,7 +316,7 @@ impl<'a> Builder<'a> {
                 tool::UnstableBookGen, tool::Tidy, tool::Linkchecker, tool::CargoTest,
                 tool::Compiletest, tool::RemoteTestServer, tool::RemoteTestClient,
                 tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc, tool::Clippy,
-                native::Llvm, tool::Rustfmt, tool::Miri),
+                native::Llvm, tool::Rustfmt, tool::Miri, native::Lld),
             Kind::Check => describe!(check::Std, check::Test, check::Rustc),
             Kind::Test => describe!(test::Tidy, test::Bootstrap, test::Ui, test::RunPass,
                 test::CompileFail, test::ParseFail, test::RunFail, test::RunPassValgrind,

src/bootstrap/cc_detect.rs

@@ -79,6 +79,9 @@ pub fn find(build: &mut Build) {
         let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false)
            .target(&target).host(&build.build);
+        if target.contains("msvc") {
+            cfg.static_crt(true);
+        }
 
         let config = build.config.target_config.get(&target);
         if let Some(cc) = config.and_then(|c| c.cc.as_ref()) {

src/bootstrap/compile.rs

@@ -747,6 +747,21 @@ fn copy_codegen_backends_to_sysroot(builder: &Builder,
     }
 }
 
+fn copy_lld_to_sysroot(builder: &Builder,
+                       target_compiler: Compiler,
+                       lld_install_root: &Path) {
+    let target = target_compiler.host;
+
+    let dst = builder.sysroot_libdir(target_compiler, target)
+        .parent()
+        .unwrap()
+        .join("bin");
+    t!(fs::create_dir_all(&dst));
+
+    let exe = exe("lld", &target);
+    copy(&lld_install_root.join("bin").join(&exe), &dst.join(&exe));
+}
+
 /// Cargo's output path for the standard library in a given stage, compiled
 /// by a particular compiler for the specified target.
 pub fn libstd_stamp(build: &Build, compiler: Compiler, target: Interned<String>) -> PathBuf {
@@ -896,6 +911,14 @@ impl Step for Assemble {
             }
         }
 
+        let lld_install = if build.config.lld_enabled && target_compiler.stage > 0 {
+            Some(builder.ensure(native::Lld {
+                target: target_compiler.host,
+            }))
+        } else {
+            None
+        };
+
         let stage = target_compiler.stage;
         let host = target_compiler.host;
         println!("Assembling stage{} compiler ({})", stage, host);
@@ -915,6 +938,9 @@ impl Step for Assemble {
         copy_codegen_backends_to_sysroot(builder,
                                          build_compiler,
                                          target_compiler);
+        if let Some(lld_install) = lld_install {
+            copy_lld_to_sysroot(builder, target_compiler, &lld_install);
+        }
 
         // Link the compiler binary itself into place
         let out_dir = build.cargo_out(build_compiler, Mode::Librustc, host);

src/bootstrap/config.rs

@@ -81,6 +81,8 @@ pub struct Config {
     pub llvm_experimental_targets: String,
     pub llvm_link_jobs: Option<u32>,
 
+    pub lld_enabled: bool,
+
     // rust codegen options
     pub rust_optimize: bool,
     pub rust_codegen_units: Option<u32>,
@@ -292,6 +294,7 @@ struct Rust {
     codegen_backends: Option<Vec<String>>,
     codegen_backends_dir: Option<String>,
     wasm_syscall: Option<bool>,
+    lld: Option<bool>,
 }
 
 /// TOML representation of how each build target is configured.
@@ -480,6 +483,7 @@ impl Config {
             set(&mut config.quiet_tests, rust.quiet_tests);
             set(&mut config.test_miri, rust.test_miri);
             set(&mut config.wasm_syscall, rust.wasm_syscall);
+            set(&mut config.lld_enabled, rust.lld);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);

src/bootstrap/configure.py

@@ -66,6 +66,7 @@ def v(*args):
 o("cargo-openssl-static", "build.openssl-static", "static openssl in cargo")
 o("profiler", "build.profiler", "build the profiler runtime")
 o("emscripten", None, "compile the emscripten backend as well as LLVM")
+o("full-tools", None, "enable all tools")
 
 # Optimization and debugging options. These may be overridden by the release
 # channel, etc.
@@ -326,6 +327,10 @@ def set(key, value):
         set('build.target', value.split(','))
     elif option.name == 'emscripten':
         set('rust.codegen-backends', ['llvm', 'emscripten'])
+    elif option.name == 'full-tools':
+        set('rust.codegen-backends', ['llvm', 'emscripten'])
+        set('rust.lld', True)
+        set('build.extended', True)
     elif option.name == 'option-checking':
         # this was handled above
         pass

src/bootstrap/dist.rs

@@ -28,7 +28,7 @@ use build_helper::output;
 
 use {Build, Compiler, Mode};
 use channel;
-use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
+use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file, exe};
 use builder::{Builder, RunConfig, ShouldRun, Step};
 use compile;
 use native;
@@ -443,6 +443,22 @@ impl Step for Rustc {
             t!(fs::create_dir_all(&backends_dst));
             cp_r(&backends_src, &backends_dst);
 
+            // Copy over lld if it's there
+            if builder.config.lld_enabled {
+                let exe = exe("lld", &compiler.host);
+                let src = builder.sysroot_libdir(compiler, host)
+                    .parent()
+                    .unwrap()
+                    .join("bin")
+                    .join(&exe);
+                let dst = image.join("lib/rustlib")
+                    .join(&*host)
+                    .join("bin")
+                    .join(&exe);
+                t!(fs::create_dir_all(&dst.parent().unwrap()));
+                copy(&src, &dst);
+            }
+
             // Man pages
             t!(fs::create_dir_all(image.join("share/man/man1")));
             let man_src = build.src.join("src/doc/man");
@@ -590,8 +606,10 @@ impl Step for Std {
         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
         cp_filtered(&src, &dst, &|path| {
-            path.file_name().and_then(|s| s.to_str()) !=
-                Some(build.config.rust_codegen_backends_dir.as_str())
+            let name = path.file_name().and_then(|s| s.to_str());
+            name != Some(build.config.rust_codegen_backends_dir.as_str()) &&
+                name != Some("bin")
+
         });
 
         let mut cmd = rust_installer(builder);

src/bootstrap/lib.rs

@@ -501,6 +501,10 @@ impl Build {
         self.out.join(&*target).join("llvm-emscripten")
     }
 
+    fn lld_out(&self, target: Interned<String>) -> PathBuf {
+        self.out.join(&*target).join("lld")
+    }
+
     /// Output directory for all documentation for a target
     fn doc_out(&self, target: Interned<String>) -> PathBuf {
         self.out.join(&*target).join("doc")
@@ -685,7 +689,9 @@ impl Build {
                                                        .and_then(|c| c.linker.as_ref()) {
             Some(linker)
         } else if target != self.config.build &&
-                  !target.contains("msvc") && !target.contains("emscripten") {
+                  !target.contains("msvc") &&
+                  !target.contains("emscripten") &&
+                  !target.contains("wasm32") {
             Some(self.cc(target))
         } else {
             None