diff --git a/Cargo.lock b/Cargo.lock index 03a4d71f67c3a..fdfa13a9066a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -266,6 +266,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "basic-toml" version = "0.1.10" @@ -382,6 +388,12 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + [[package]] name = "camino" version = "1.2.1" @@ -1726,6 +1738,22 @@ dependencies = [ "match_token", ] +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humansize" version = "2.1.3" @@ -2762,6 +2790,7 @@ dependencies = [ "tabled", "tar", "tempfile", + "ureq", "xz2", ] @@ -3311,6 +3340,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "run_make_support" version = "0.0.0" @@ -4865,7 +4908,7 @@ version = "0.0.0" dependencies = [ "arrayvec", "askama", - "base64", + "base64 0.21.7", "expect-test", "indexmap", "itertools", @@ -4982,6 +5025,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -5389,6 +5467,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" @@ -6065,6 +6149,41 @@ dependencies = [ "tidy", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.7" @@ -6345,6 +6464,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6526,6 +6654,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -6863,6 +7000,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerotrie" version = "0.2.3" diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index f4051ae67d7c0..f922ba5f8f2cd 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,4 @@ tempfile = "3.5" derive_builder = "0.20" clap = { version = "4", features = ["derive"] } tabled = { version = "0.15", default-features = false, features = ["std"] } +ureq = "3.1.4" diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 48b25f235dd6a..eed70ffc736e2 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,7 +1,12 @@ +#![allow(unused, warnings)] + use anyhow::Context; +use build_helper::ci::CiEnv; +use build_helper::git::{GitConfig, get_closest_upstream_commit}; use camino::{Utf8Path, Utf8PathBuf}; use clap::Parser; use log::LevelFilter; +use ureq::http::StatusCode; use utils::io; use crate::bolt::{bolt_optimize, with_bolt_instrumented}; @@ -10,11 +15,11 @@ use crate::exec::{Bootstrap, cmd}; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{ - gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks, - rustc_benchmarks, + BoltProfile, LlvmPGOProfile, RustcPGOProfile, gather_bolt_profiles, gather_llvm_profiles, + gather_rustc_profiles, llvm_benchmarks, rustc_benchmarks, }; use crate::utils::artifact_size::print_binary_sizes; -use crate::utils::io::{copy_directory, reset_directory}; +use crate::utils::io::{copy_directory, find_file_in_dir, reset_directory}; use crate::utils::{ clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group, write_timer_to_summary, @@ -464,7 +469,47 @@ fn main() -> anyhow::Result<()> { let mut timer = Timer::new(); - let result = execute_pipeline(&env, &mut timer, build_args); + let parent_sha = get_closest_upstream_commit( + None, + &GitConfig { nightly_branch: "main", git_merge_commit_email: "bors@rust-lang.org" }, + CiEnv::GitHubActions, + ) + .unwrap() + .unwrap(); + + let url = format!( + "https://ci-artifacts.rust-lang.org/rustc-builds/{parent_sha}/reproducible-artifacts-nightly-x86_64-unknown-linux-gnu.tar.xz" + ); + let response = ureq::get(url).call()?; + if !response.status().is_success() { + return Err(anyhow::anyhow!("Error while downloading PGO: {}", response.status())); + } + let mut body = response.into_body(); + let mut archive = tar::Archive::new(xz::read::XzDecoder::new(body.as_reader())); + + let pgo_dir = Utf8Path::new("pgo-profiles"); + archive.unpack(pgo_dir)?; + let dir = find_file_in_dir(pgo_dir, "reproducible-artifacts", "")?; + let dir = dir.join("reproducible-artifacts"); + let llvm_pgo_profile = LlvmPGOProfile(dir.join("llvm-pgo.profdata")); + let rustc_pgo_profile = RustcPGOProfile(dir.join("rustc-pgo.profdata")); + let bolt_profiles = + [BoltProfile(dir.join("LLVM-bolt.profdata")), BoltProfile(dir.join("rustc-bolt.profdata"))]; + + let mut dist = Bootstrap::dist(&env, &build_args) + .llvm_pgo_optimize(Some(&llvm_pgo_profile)) + .rustc_pgo_optimize(&rustc_pgo_profile); + + // for bolt_profile in bolt_profiles { + // dist = dist.with_bolt_profile(Some(bolt_profile)); + // } + // dist = dist.with_llvm_bolt_ldflags().with_rustc_bolt_ldflags(); + + // Final stage: Assemble the dist artifacts + // The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused. + timer.section("Stage 5 (final build)", |stage| dist.run(stage))?; + + // let result = execute_pipeline(&env, &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") { @@ -472,7 +517,7 @@ fn main() -> anyhow::Result<()> { } print_free_disk_space()?; - result.context("Optimized build pipeline has failed")?; + // result.context("Optimized build pipeline has failed")?; print_binary_sizes(&env)?; Ok(())