Use rustc target metadata for build-manifest target lists

compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs

@@ -17,7 +17,7 @@ pub(crate) fn target() -> Target {
         metadata: TargetMetadata {
             description: Some("ARM64 MinGW (Windows 10+), LLVM ABI".into()),
             tier: Some(2),
-            host_tools: Some(false),
+            host_tools: Some(true),
             std: Some(true),
         },
         pointer_width: 64,

compiler/rustc_target/src/spec/targets/i586_unknown_linux_gnu.rs

@@ -5,5 +5,11 @@ pub(crate) fn target() -> Target {
     base.rustc_abi = None; // overwrite the SSE2 ABI set by the base target
     base.cpu = "pentium".into();
     base.llvm_target = "i586-unknown-linux-gnu".into();
+    base.metadata = crate::spec::TargetMetadata {
+        description: Some("32-bit Linux (kernel 3.2, glibc 2.17+)".into()),
+        tier: Some(2),
+        host_tools: Some(false),
+        std: Some(true),
+    };
     base
 }

compiler/rustc_target/src/spec/targets/sparcv9_sun_solaris.rs

@@ -18,7 +18,7 @@ pub(crate) fn target() -> Target {
         metadata: TargetMetadata {
             description: Some("SPARC Solaris 11.4".into()),
             tier: Some(2),
-            host_tools: Some(false),
+            host_tools: Some(true),
             std: Some(true),
         },
         pointer_width: 64,

compiler/rustc_target/src/spec/targets/wasm32_wasip2.rs

@@ -63,7 +63,7 @@ pub(crate) fn target() -> Target {
         llvm_target: "wasm32-wasip2".into(),
         metadata: TargetMetadata {
             description: Some("WebAssembly".into()),
-            tier: Some(3),
+            tier: Some(2),
             host_tools: Some(false),
             std: Some(true),
         },

compiler/rustc_target/src/spec/targets/wasm32_wasip3.rs

@@ -15,6 +15,12 @@ pub(crate) fn target() -> Target {
     // and this may grow over time as more features are supported.
     let mut target = super::wasm32_wasip2::target();
     target.llvm_target = "wasm32-wasip3".into();
+    target.metadata = crate::spec::TargetMetadata {
+        description: Some("WebAssembly".into()),
+        tier: Some(3),
+        host_tools: Some(false),
+        std: Some(true),
+    };
     target.options.env = Env::P3;
     target
 }

compiler/rustc_target/src/spec/targets/x86_64_pc_solaris.rs

@@ -20,7 +20,7 @@ pub(crate) fn target() -> Target {
         metadata: TargetMetadata {
             description: Some("64-bit Solaris 11.4".into()),
             tier: Some(2),
-            host_tools: Some(false),
+            host_tools: Some(true),
             std: Some(true),
         },
         pointer_width: 64,

compiler/rustc_target/src/spec/targets/x86_64_pc_windows_gnullvm.rs

@@ -14,7 +14,7 @@ pub(crate) fn target() -> Target {
         metadata: TargetMetadata {
             description: Some("64-bit x86 MinGW (Windows 10+), LLVM ABI".into()),
             tier: Some(2),
-            host_tools: Some(false),
+            host_tools: Some(true),
             std: Some(true),
         },
         pointer_width: 64,

src/bootstrap/src/core/build_steps/dist.rs

@@ -22,7 +22,7 @@ use tracing::instrument;
 use crate::core::build_steps::compile::{get_codegen_backend_file, normalize_codegen_backend_name};
 use crate::core::build_steps::doc::DocumentationFormat;
 use crate::core::build_steps::tool::{
-    self, RustcPrivateCompilers, Tool, ToolTargetBuildMode, get_tool_target_compiler,
+    self, RustcPrivateCompilers, ToolTargetBuildMode, get_tool_target_compiler,
 };
 use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor};
 use crate::core::build_steps::{compile, llvm};
@@ -2695,10 +2695,14 @@ impl Step for BuildManifest {
     }
 
     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
-        let build_manifest = builder.tool_exe(Tool::BuildManifest);
+        // FIXME: Should BuildManifest actually be built for `self.target`?
+        // Today CI only builds this step where that matches the host_target so it doesn't matter
+        // today.
+        let build_manifest =
+            builder.ensure(tool::BuildManifest::new(builder, builder.config.host_target));
 
         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
-        tarball.add_file(&build_manifest, "bin", FileType::Executable);
+        tarball.add_file(&build_manifest.tool_path, "bin", FileType::Executable);
         tarball.generate()
     }
 

src/bootstrap/src/core/build_steps/run.rs

@@ -37,7 +37,9 @@ impl Step for BuildManifest {
     fn run(self, builder: &Builder<'_>) {
         // This gets called by `promote-release`
         // (https://github.com/rust-lang/promote-release).
-        let mut cmd = builder.tool_cmd(Tool::BuildManifest);
+        let mut cmd = command(
+            builder.ensure(tool::BuildManifest::new(builder, builder.config.host_target)).tool_path,
+        );
         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
             panic!("\n\nfailed to specify `dist.sign-folder` in `bootstrap.toml`\n\n")
         });

src/bootstrap/src/core/build_steps/tool.rs

@@ -485,7 +485,6 @@ bootstrap_tool!(
     Linkchecker, "src/tools/linkchecker", "linkchecker";
     CargoTest, "src/tools/cargotest", "cargotest";
     Compiletest, "src/tools/compiletest", "compiletest";
-    BuildManifest, "src/tools/build-manifest", "build-manifest";
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
     RustInstaller, "src/tools/rust-installer", "rust-installer";
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
@@ -1268,6 +1267,52 @@ impl Step for LibcxxVersionTool {
     }
 }
 
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct BuildManifest {
+    compiler: Compiler,
+    target: TargetSelection,
+}
+
+impl BuildManifest {
+    pub fn new(builder: &Builder<'_>, target: TargetSelection) -> Self {
+        BuildManifest { compiler: builder.compiler(1, builder.config.host_target), target }
+    }
+}
+
+impl Step for BuildManifest {
+    type Output = ToolBuildResult;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/build-manifest")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(BuildManifest::new(run.builder, run.target));
+    }
+
+    fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
+        // Building with the beta compiler will produce a broken build-manifest that doesn't support
+        // recently stabilized targets/hosts.
+        assert!(self.compiler.stage != 0);
+        builder.ensure(ToolBuild {
+            build_compiler: self.compiler,
+            target: self.target,
+            tool: "build-manifest",
+            mode: Mode::ToolStd,
+            path: "src/tools/build-manifest",
+            source_type: SourceType::InTree,
+            extra_features: vec![],
+            allow_features: "",
+            cargo_args: vec![],
+            artifact_kind: ToolArtifactKind::Binary,
+        })
+    }
+
+    fn metadata(&self) -> Option<StepMetadata> {
+        Some(StepMetadata::build("build-manifest", self.target).built_by(self.compiler))
+    }
+}
+
 /// Represents which compilers are involved in the compilation of a tool
 /// that depends on compiler internals (`rustc_private`).
 /// Their compilation looks like this:

src/tools/build-manifest/Cargo.toml

@@ -14,3 +14,7 @@ tar = "0.4.29"
 sha2 = "0.10.1"
 rayon = "1.5.1"
 hex = "0.4.2"
+
+[build-dependencies]
+serde = "1"
+serde_json = "1"

src/tools/build-manifest/README.md

@@ -2,7 +2,13 @@
 
 This tool generates the manifests uploaded to static.rust-lang.org and used by rustup.
 You can see a full list of all manifests at <https://static.rust-lang.org/manifests.txt>.
-This listing is updated by <https://github.com/rust-lang/generate-manifest-list> every 7 days.
+
+We auto-generate the host targets (those with full compiler toolchains) and
+target targets (a superset of hosts, some of which only support std) through
+`build.rs`, which internally uses a stage 1 rustc to produce the target list
+and uses the `TargetMetadata` to determine whether host tools are expected and
+whether artifacts are expected. This list is not currently verified against the
+actually produced artifacts by CI, though that may change in the future.
 
 This gets called by `promote-release` <https://github.com/rust-lang/promote-release>. `promote-release` downloads a pre-built binary of `build-manifest` which is generated in the dist-x86_64-linux builder and uploaded to s3.
 

src/tools/build-manifest/build.rs

@@ -0,0 +1,76 @@
+use std::fmt::Write;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+
+#[derive(Default, Debug)]
+pub(crate) struct RustcTargets {
+    /// Targets with host tool artifacts.
+    pub(crate) hosts: Vec<String>,
+
+    /// All targets we distribute some artifacts for (superset of `hosts`).
+    pub(crate) targets: Vec<String>,
+}
+
+fn collect_rustc_targets() -> RustcTargets {
+    let rustc_path = std::env::var("RUSTC").expect("RUSTC set");
+    let output = Command::new(&rustc_path)
+        .arg("--print=all-target-specs-json")
+        .env("RUSTC_BOOTSTRAP", "1")
+        .arg("-Zunstable-options")
+        .stderr(Stdio::inherit())
+        .output()
+        .unwrap();
+    assert!(output.status.success());
+    let json: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
+
+    let mut rustc_targets = RustcTargets::default();
+    for (target, json) in json.as_object().unwrap().iter() {
+        let Some(tier) = json["metadata"]["tier"].as_u64() else {
+            eprintln!("skipping {target}: no tier in metadata");
+            continue;
+        };
+        let host_tools: Option<bool> =
+            serde_json::from_value(json["metadata"]["host_tools"].clone()).unwrap();
+
+        if !(tier == 1 || tier == 2) {
+            eprintln!("ignoring {target}: tier {tier} insufficient for target to be in manifest");
+            continue;
+        }
+
+        if host_tools == Some(true) {
+            rustc_targets.hosts.push(target.to_owned());
+            rustc_targets.targets.push(target.to_owned());
+        } else {
+            rustc_targets.targets.push(target.to_owned());
+        }
+    }
+
+    rustc_targets
+}
+
+fn main() {
+    let targets = collect_rustc_targets();
+
+    // Verify we ended up with a reasonable target list.
+    assert!(targets.hosts.len() >= 10);
+    assert!(targets.targets.len() >= 30);
+    assert!(targets.hosts.iter().any(|e| e == "x86_64-unknown-linux-gnu"));
+    assert!(targets.targets.iter().any(|e| e == "x86_64-unknown-linux-gnu"));
+
+    let mut output = String::new();
+
+    writeln!(output, "static HOSTS: &[&str] = &[").unwrap();
+    for host in targets.hosts {
+        writeln!(output, "    {:?},", host).unwrap();
+    }
+    writeln!(output, "];").unwrap();
+
+    writeln!(output, "static TARGETS: &[&str] = &[").unwrap();
+    for target in targets.targets {
+        writeln!(output, "    {:?},", target).unwrap();
+    }
+    writeln!(output, "];").unwrap();
+
+    std::fs::write(PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("targets.rs"), output)
+        .unwrap();
+}