Auto merge of #116035 - lqd:mcp-510-target-specs, r=petrochenkov

Allow target specs to use an LLD flavor, and self-contained linking components

This PR allows:
- target specs to use an LLD linker-flavor: this is needed to switch `x86_64-unknown-linux-gnu` to using LLD, and is currently not possible because the current flavor json serialization fails to roundtrip on the modern linker-flavors. This can e.g. be seen in #115622 (comment) which explains where an `Lld::Yes` is ultimately deserialized into an `Lld::No`.
- target specs to declare self-contained linking components: this is needed to switch `x86_64-unknown-linux-gnu` to using `rust-lld`
- adds an end-to-end test of a custom target json simulating `x86_64-unknown-linux-gnu` being switched to using `rust-lld`
- disables codegen backends from participating because they don't support `-Zgcc-ld=lld` which is the basis of mcp510.

r? `@petrochenkov:` if the approach discussed #115622 (comment) and on zulip would work for you: basically, see if we can emit only modern linker flavors in the json specs, but accept both old and new flavors while reading them, to fix the roundtrip issue.

The backwards compatible `LinkSelfContainedDefault` variants are still serialized and deserialized in `crt-objects-fallback`, while the spec equivalent of e.g. `-Clink-self-contained=+linker` is serialized into a different json object (with future-proofing to incorporate `crt-objects-fallback`  in the future).

---

I've been test-driving this in #113382 to test actually switching `x86_64-unknown-linux-gnu`  to `rust-lld` (and fix what needs to be fixed in CI, bootstrap, etc), and it seems to work fine.

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -22,7 +22,9 @@ use rustc_session::utils::NativeLibKind;
 /// need out of the shared crate context before we get rid of it.
 use rustc_session::{filesearch, Session};
 use rustc_span::symbol::Symbol;
-use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
+use rustc_target::spec::crt_objects::CrtObjects;
+use rustc_target::spec::LinkSelfContainedComponents;
+use rustc_target::spec::LinkSelfContainedDefault;
 use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, Lld, PanicStrategy};
 use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo};
 
@@ -728,6 +730,7 @@ fn link_natively<'a>(
 ) -> Result<(), ErrorGuaranteed> {
     info!("preparing {:?} to {:?}", crate_type, out_filename);
     let (linker_path, flavor) = linker_and_flavor(sess);
+    let self_contained_components = self_contained_components(sess, crate_type);
     let mut cmd = linker_with_args(
         &linker_path,
         flavor,
@@ -737,6 +740,7 @@ fn link_natively<'a>(
         tmpdir,
         out_filename,
         codegen_results,
+        self_contained_components,
     )?;
 
     linker::disable_localization(&mut cmd);
@@ -812,14 +816,14 @@ fn link_natively<'a>(
                 "Linker does not support -static-pie command line option. Retrying with -static instead."
             );
             // Mirror `add_(pre,post)_link_objects` to replace CRT objects.
-            let self_contained = self_contained(sess, crate_type);
+            let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
             let opts = &sess.target;
-            let pre_objects = if self_contained {
+            let pre_objects = if self_contained_crt_objects {
                 &opts.pre_link_objects_self_contained
             } else {
                 &opts.pre_link_objects
             };
-            let post_objects = if self_contained {
+            let post_objects = if self_contained_crt_objects {
                 &opts.post_link_objects_self_contained
             } else {
                 &opts.post_link_objects
@@ -830,7 +834,9 @@ fn link_natively<'a>(
                     .iter()
                     .copied()
                     .flatten()
-                    .map(|obj| get_object_file_path(sess, obj, self_contained).into_os_string())
+                    .map(|obj| {
+                        get_object_file_path(sess, obj, self_contained_crt_objects).into_os_string()
+                    })
                     .collect::<Vec<_>>()
             };
             let pre_objects_static_pie = get_objects(pre_objects, LinkOutputKind::StaticPicExe);
@@ -1710,26 +1716,43 @@ fn detect_self_contained_mingw(sess: &Session) -> bool {
 /// Various toolchain components used during linking are used from rustc distribution
 /// instead of being found somewhere on the host system.
 /// We only provide such support for a very limited number of targets.
-fn self_contained(sess: &Session, crate_type: CrateType) -> bool {
-    if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set {
-        if sess.target.link_self_contained == LinkSelfContainedDefault::False {
-            sess.emit_err(errors::UnsupportedLinkSelfContained);
-        }
-        return self_contained;
-    }
+fn self_contained_components(sess: &Session, crate_type: CrateType) -> LinkSelfContainedComponents {
+    // Turn the backwards compatible bool values for `self_contained` into fully inferred
+    // `LinkSelfContainedComponents`.
+    let self_contained =
+        if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set {
+            // Emit an error if the user requested self-contained mode on the CLI but the target
+            // explicitly refuses it.
+            if sess.target.link_self_contained.is_disabled() {
+                sess.emit_err(errors::UnsupportedLinkSelfContained);
+            }
+            self_contained
+        } else {
+            match sess.target.link_self_contained {
+                LinkSelfContainedDefault::False => false,
+                LinkSelfContainedDefault::True => true,
+
+                LinkSelfContainedDefault::WithComponents(components) => {
+                    // For target specs with explicitly enabled components, we can return them
+                    // directly.
+                    return components;
+                }
 
-    match sess.target.link_self_contained {
-        LinkSelfContainedDefault::False => false,
-        LinkSelfContainedDefault::True => true,
-        // FIXME: Find a better heuristic for "native musl toolchain is available",
-        // based on host and linker path, for example.
-        // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
-        LinkSelfContainedDefault::Musl => sess.crt_static(Some(crate_type)),
-        LinkSelfContainedDefault::Mingw => {
-            sess.host == sess.target
-                && sess.target.vendor != "uwp"
-                && detect_self_contained_mingw(&sess)
-        }
+                // FIXME: Find a better heuristic for "native musl toolchain is available",
+                // based on host and linker path, for example.
+                // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
+                LinkSelfContainedDefault::InferredForMusl => sess.crt_static(Some(crate_type)),
+                LinkSelfContainedDefault::InferredForMingw => {
+                    sess.host == sess.target
+                        && sess.target.vendor != "uwp"
+                        && detect_self_contained_mingw(&sess)
+                }
+            }
+        };
+    if self_contained {
+        LinkSelfContainedComponents::all()
+    } else {
+        LinkSelfContainedComponents::empty()
     }
 }
 
@@ -2030,13 +2053,14 @@ fn linker_with_args<'a>(
     tmpdir: &Path,
     out_filename: &Path,
     codegen_results: &CodegenResults,
+    self_contained_components: LinkSelfContainedComponents,
 ) -> Result<Command, ErrorGuaranteed> {
-    let self_contained = self_contained(sess, crate_type);
+    let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
     let cmd = &mut *super::linker::get_linker(
         sess,
         path,
         flavor,
-        self_contained,
+        self_contained_components.are_any_components_enabled(),
         &codegen_results.crate_info.target_cpu,
     );
     let link_output_kind = link_output_kind(sess, crate_type);
@@ -2063,7 +2087,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Pre-link CRT objects.
-    add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained);
+    add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained_crt_objects);
 
     add_linked_symbol_object(
         cmd,
@@ -2206,7 +2230,7 @@ fn linker_with_args<'a>(
         cmd,
         sess,
         link_output_kind,
-        self_contained,
+        self_contained_components,
         flavor,
         crate_type,
         codegen_results,
@@ -2222,7 +2246,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Post-link CRT objects.
-    add_post_link_objects(cmd, sess, link_output_kind, self_contained);
+    add_post_link_objects(cmd, sess, link_output_kind, self_contained_crt_objects);
 
     // ------------ Late order-dependent options ------------
 
@@ -2239,15 +2263,15 @@ fn add_order_independent_options(
     cmd: &mut dyn Linker,
     sess: &Session,
     link_output_kind: LinkOutputKind,
-    self_contained: bool,
+    self_contained_components: LinkSelfContainedComponents,
     flavor: LinkerFlavor,
     crate_type: CrateType,
     codegen_results: &CodegenResults,
     out_filename: &Path,
     tmpdir: &Path,
 ) {
     // Take care of the flavors and CLI options requesting the `lld` linker.
-    add_lld_args(cmd, sess, flavor);
+    add_lld_args(cmd, sess, flavor, self_contained_components);
 
     add_apple_sdk(cmd, sess, flavor);
 
@@ -2272,7 +2296,7 @@ fn add_order_independent_options(
     // Make the binary compatible with data execution prevention schemes.
     cmd.add_no_exec();
 
-    if self_contained {
+    if self_contained_components.is_crt_objects_enabled() {
         cmd.no_crt_objects();
     }
 
@@ -2303,7 +2327,7 @@ fn add_order_independent_options(
 
     cmd.linker_plugin_lto();
 
-    add_library_search_dirs(cmd, sess, self_contained);
+    add_library_search_dirs(cmd, sess, self_contained_components.are_any_components_enabled());
 
     cmd.output_filename(out_filename);
 
@@ -2953,8 +2977,16 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootErro
 /// invoke it:
 /// - when the self-contained linker flag is active: the build of `lld` distributed with rustc,
 /// - or any `lld` available to `cc`.
-fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
-    debug!("add_lld_args requested, flavor: '{flavor:?}'");
+fn add_lld_args(
+    cmd: &mut dyn Linker,
+    sess: &Session,
+    flavor: LinkerFlavor,
+    self_contained_components: LinkSelfContainedComponents,
+) {
+    debug!(
+        "add_lld_args requested, flavor: '{:?}', target self-contained components: {:?}",
+        flavor, self_contained_components,
+    );
 
     // If the flavor doesn't use a C/C++ compiler to invoke the linker, or doesn't opt in to `lld`,
     // we don't need to do anything.
@@ -2963,9 +2995,32 @@ fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
     }
 
     // 1. Implement the "self-contained" part of this feature by adding rustc distribution
-    // directories to the tool's search path:
-    // - if the self-contained linker is enabled on the CLI.
-    if sess.opts.cg.link_self_contained.is_linker_enabled() {
+    // directories to the tool's search path, depending on a mix between what users can specify on
+    // the CLI, and what the target spec enables (as it can't disable components):
+    // - if the self-contained linker is enabled on the CLI or by the target spec,
+    // - and if the self-contained linker is not disabled on the CLI.
+    let self_contained_cli = sess.opts.cg.link_self_contained.is_linker_enabled();
+    let self_contained_target = self_contained_components.is_linker_enabled();
+
+    // FIXME: in the future, codegen backends may need to have more control over this process: they
+    // don't always support all the features the linker expects here, and vice versa. For example,
+    // at the time of writing this, lld expects a newer style of aarch64 TLS relocations that
+    // cranelift doesn't implement yet. That in turn can impact whether linking would succeed on
+    // such a target when using the `cg_clif` backend and lld.
+    //
+    // Until interactions between backends and linker features are expressible, we limit target
+    // specs to opt-in to lld only when we're on the llvm backend, where it's expected to work and
+    // tested on CI. As usual, the CLI still has precedence over this, so that users and developers
+    // can still override this default when needed (e.g. for tests).
+    let uses_llvm_backend =
+        matches!(sess.opts.unstable_opts.codegen_backend.as_deref(), None | Some("llvm"));
+    if !uses_llvm_backend && !self_contained_cli && sess.opts.cg.linker_flavor.is_none() {
+        // We bail if we're not using llvm and lld was not explicitly requested on the CLI.
+        return;
+    }
+
+    let self_contained_linker = self_contained_cli || self_contained_target;
+    if self_contained_linker && !sess.opts.cg.link_self_contained.is_linker_disabled() {
         for path in sess.get_tools_search_paths(false) {
             cmd.arg({
                 let mut arg = OsString::from("-B");

compiler/rustc_target/src/spec/crt_objects.rs

@@ -40,11 +40,9 @@
 //! but not gcc's. As a result rustc cannot link with C++ static libraries (#36710)
 //! when linking in self-contained mode.
 
-use crate::json::{Json, ToJson};
 use crate::spec::LinkOutputKind;
 use std::borrow::Cow;
 use std::collections::BTreeMap;
-use std::str::FromStr;
 
 pub type CrtObjects = BTreeMap<LinkOutputKind, Vec<Cow<'static, str>>>;
 
@@ -123,39 +121,3 @@ pub(super) fn pre_wasi_self_contained() -> CrtObjects {
 pub(super) fn post_wasi_self_contained() -> CrtObjects {
     new(&[])
 }
-
-/// Which logic to use to determine whether to use self-contained linking mode
-/// if `-Clink-self-contained` is not specified explicitly.
-#[derive(Clone, Copy, PartialEq, Hash, Debug)]
-pub enum LinkSelfContainedDefault {
-    False,
-    True,
-    Musl,
-    Mingw,
-}
-
-impl FromStr for LinkSelfContainedDefault {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<LinkSelfContainedDefault, ()> {
-        Ok(match s {
-            "false" => LinkSelfContainedDefault::False,
-            "true" | "wasm" => LinkSelfContainedDefault::True,
-            "musl" => LinkSelfContainedDefault::Musl,
-            "mingw" => LinkSelfContainedDefault::Mingw,
-            _ => return Err(()),
-        })
-    }
-}
-
-impl ToJson for LinkSelfContainedDefault {
-    fn to_json(&self) -> Json {
-        match *self {
-            LinkSelfContainedDefault::False => "false",
-            LinkSelfContainedDefault::True => "true",
-            LinkSelfContainedDefault::Musl => "musl",
-            LinkSelfContainedDefault::Mingw => "mingw",
-        }
-        .to_json()
-    }
-}

compiler/rustc_target/src/spec/linux_musl_base.rs

@@ -1,13 +1,13 @@
-use crate::spec::crt_objects::{self, LinkSelfContainedDefault};
-use crate::spec::TargetOptions;
+use crate::spec::crt_objects;
+use crate::spec::{LinkSelfContainedDefault, TargetOptions};
 
 pub fn opts() -> TargetOptions {
     let mut base = super::linux_base::opts();
 
     base.env = "musl".into();
     base.pre_link_objects_self_contained = crt_objects::pre_musl_self_contained();
     base.post_link_objects_self_contained = crt_objects::post_musl_self_contained();
-    base.link_self_contained = LinkSelfContainedDefault::Musl;
+    base.link_self_contained = LinkSelfContainedDefault::InferredForMusl;
 
     // These targets statically link libc by default
     base.crt_static_default = true;