From 588db24344dc2b626bb050067e9e6cda2de3bc59 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 14:41:18 +0100 Subject: [PATCH 01/10] Correctly handle `--use-system-gcc` --- build_system/src/build.rs | 4 ++-- build_system/src/cargo.rs | 2 +- build_system/src/config.rs | 35 ++++++++++------------------- build_system/src/test.rs | 45 +++++++++++++++++++------------------- build_system/src/utils.rs | 6 ++++- 5 files changed, 42 insertions(+), 50 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index efae5a46b04f9..308ad34654941 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -207,7 +207,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { } run_command_with_output_and_env(&command, None, Some(&env))?; - args.config_info.setup(&mut env, None)?; + args.config_info.setup(&mut env, false)?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); @@ -229,7 +229,7 @@ pub fn run() -> Result<(), String> { Some(args) => args, None => return Ok(()), }; - args.config_info.setup_gcc_path(None)?; + args.config_info.setup_gcc_path()?; build_codegen(&mut args)?; Ok(()) } diff --git a/build_system/src/cargo.rs b/build_system/src/cargo.rs index 67b301d9aa646..1cfcdba6b1cd1 100644 --- a/build_system/src/cargo.rs +++ b/build_system/src/cargo.rs @@ -77,7 +77,7 @@ pub fn run() -> Result<(), String> { })?; let mut env: HashMap = std::env::vars().collect(); - ConfigInfo::default().setup(&mut env, None)?; + ConfigInfo::default().setup(&mut env, false)?; let toolchain = get_toolchain()?; let toolchain_version = rustc_toolchain_version_info(&toolchain)?; diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 5ba6233617ec5..49782fc64ef70 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -159,30 +159,17 @@ impl ConfigInfo { command } - pub fn setup_gcc_path(&mut self, override_gcc_path: Option<&str>) -> Result<(), String> { + pub fn setup_gcc_path(&mut self) -> Result<(), String> { let ConfigFile { gcc_path, .. } = ConfigFile::new(self.config_file.as_deref())?; - self.gcc_path = match override_gcc_path { - Some(path) => { - if gcc_path.is_some() { - println!( - "overriding setting from `{}`", - self.config_file.as_deref().unwrap_or("config.toml") - ); - } - path.to_string() - } + self.gcc_path = match gcc_path { + Some(path) => path, + // FIXME: Once we support "download", rewrite this. None => { - match gcc_path { - Some(path) => path, - // FIXME: Once we support "download", rewrite this. - None => { - return Err(format!( - "missing `gcc-path` value from `{}`", - self.config_file.as_deref().unwrap_or("config.toml"), - )) - } - } + return Err(format!( + "missing `gcc-path` value from `{}`", + self.config_file.as_deref().unwrap_or("config.toml"), + )) } }; Ok(()) @@ -191,12 +178,12 @@ impl ConfigInfo { pub fn setup( &mut self, env: &mut HashMap, - override_gcc_path: Option<&str>, + use_system_gcc: bool, ) -> Result<(), String> { env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); - if self.gcc_path.is_empty() || override_gcc_path.is_some() { - self.setup_gcc_path(override_gcc_path)?; + if self.gcc_path.is_empty() && !use_system_gcc { + self.setup_gcc_path()?; } env.insert("GCC_PATH".to_string(), self.gcc_path.clone()); diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 1cacd6efc7fee..806e18431c46d 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -31,7 +31,10 @@ fn get_runners() -> Runners { "--test-failing-rustc", ("Run failing rustc tests", test_failing_rustc), ); - runners.insert("--projects", ("Run the tests of popular crates", test_projects)); + runners.insert( + "--projects", + ("Run the tests of popular crates", test_projects), + ); runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); runners.insert("--clean", ("Empty cargo target directory", clean)); runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); @@ -109,7 +112,7 @@ fn show_usage() { struct TestArg { no_default_features: bool, build_only: bool, - gcc_path: Option, + use_system_gcc: bool, runners: BTreeSet, flags: Vec, backend: Option, @@ -121,7 +124,6 @@ struct TestArg { impl TestArg { fn new() -> Result, String> { - let mut use_system_gcc = false; let mut test_arg = Self::default(); // We skip binary name and the `test` command. @@ -147,7 +149,10 @@ impl TestArg { return Err("Expected an argument after `--features`, found nothing".into()) } }, - "--use-system-gcc" => use_system_gcc = true, + "--use-system-gcc" => { + println!("Using system GCC"); + test_arg.use_system_gcc = true; + } "--build-only" => test_arg.build_only = true, "--use-backend" => match args.next() { Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend), @@ -180,11 +185,6 @@ impl TestArg { } } } - - if use_system_gcc { - println!("Using system GCC"); - test_arg.gcc_path = Some("gcc".to_string()); - } } match (test_arg.current_part, test_arg.nb_parts) { (Some(_), Some(_)) | (None, None) => {} @@ -703,7 +703,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { //"https://github.com/rust-lang/cargo", // TODO: very slow, only run on master? ]; - let run_tests = |projects_path, iter: &mut dyn Iterator| -> Result<(), String> { + let run_tests = |projects_path, iter: &mut dyn Iterator| -> Result<(), String> { for project in iter { let clone_result = git_clone(project, Some(projects_path), true)?; let repo_path = Path::new(&clone_result.repo_dir); @@ -727,8 +727,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { let start = current_part * count; // We remove the projects we don't want to test. run_tests(projects_path, &mut projects.iter().skip(start).take(count))?; - } - else { + } else { run_tests(projects_path, &mut projects.iter())?; } @@ -1166,15 +1165,17 @@ pub fn run() -> Result<(), String> { }; let mut env: HashMap = std::env::vars().collect(); - args.config_info.setup_gcc_path(None)?; - env.insert( - "LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); - env.insert( - "LD_LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); + if !args.use_system_gcc { + args.config_info.setup_gcc_path()?; + env.insert( + "LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + env.insert( + "LD_LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + } build_if_no_backend(&env, &args)?; if args.build_only { @@ -1182,7 +1183,7 @@ pub fn run() -> Result<(), String> { return Ok(()); } - args.config_info.setup(&mut env, args.gcc_path.as_deref())?; + args.config_info.setup(&mut env, args.use_system_gcc)?; if args.runners.is_empty() { run_all(&env, &args)?; diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index b288eff94a5d4..046008ae1a247 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -254,7 +254,11 @@ pub struct CloneResult { pub repo_dir: String, } -pub fn git_clone(to_clone: &str, dest: Option<&Path>, shallow_clone: bool) -> Result { +pub fn git_clone( + to_clone: &str, + dest: Option<&Path>, + shallow_clone: bool, +) -> Result { let repo_name = to_clone.split('/').last().unwrap(); let repo_name = match repo_name.strip_suffix(".git") { Some(n) => n.to_string(), From eee04a48d9b0ba2ca7e18c6465c51a63feed8e08 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 18:07:05 +0100 Subject: [PATCH 02/10] Add support for "download" --- build_system/src/config.rs | 201 ++++++++++++++++++++++++++++++++----- libgccjit.version | 1 + 2 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 libgccjit.version diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 49782fc64ef70..0201e3509dcd3 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,9 +1,9 @@ -use crate::utils::{get_os_name, rustc_version_info, split_args}; +use crate::utils::{get_os_name, run_command_with_output, rustc_version_info, split_args}; use std::collections::HashMap; use std::env as std_env; use std::ffi::OsStr; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use boml::{types::TomlValue, Toml}; @@ -23,8 +23,12 @@ impl Channel { } } -fn failed_config_parsing(config_file: &str, err: &str) -> Result { - Err(format!("Failed to parse `{}`: {}", config_file, err)) +fn failed_config_parsing(config_file: &Path, err: &str) -> Result { + Err(format!( + "Failed to parse `{}`: {}", + config_file.display(), + err + )) } #[derive(Default)] @@ -34,12 +38,11 @@ pub struct ConfigFile { } impl ConfigFile { - pub fn new(config_file: Option<&str>) -> Result { - let config_file = config_file.unwrap_or("config.toml"); + pub fn new(config_file: &Path) -> Result { let content = fs::read_to_string(config_file).map_err(|_| { format!( "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project", - config_file, + config_file.display(), ) })?; let toml = Toml::parse(&content).map_err(|err| { @@ -70,19 +73,30 @@ impl ConfigFile { _ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)), } } - if config.gcc_path.is_none() && config.download_gccjit.is_none() { - return failed_config_parsing( - config_file, - "At least one of `gcc-path` or `download-gccjit` value must be set", - ); - } - if let Some(gcc_path) = config.gcc_path.as_mut() { - let path = Path::new(gcc_path); - *gcc_path = path - .canonicalize() - .map_err(|err| format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err))? - .display() - .to_string(); + match (config.gcc_path.as_mut(), config.download_gccjit) { + (None, None | Some(false)) => { + return failed_config_parsing( + config_file, + "At least one of `gcc-path` or `download-gccjit` value must be set", + ) + } + (Some(_), Some(true)) => { + println!( + "WARNING: both `gcc-path` and `download-gccjit` arguments are used, \ + ignoring `gcc-path`" + ); + } + (Some(gcc_path), _) => { + let path = Path::new(gcc_path); + *gcc_path = path + .canonicalize() + .map_err(|err| { + format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err) + })? + .display() + .to_string(); + } + _ => {} } Ok(config) } @@ -104,6 +118,7 @@ pub struct ConfigInfo { pub sysroot_path: String, pub gcc_path: String, config_file: Option, + cg_gcc_path: Option, } impl ConfigInfo { @@ -146,6 +161,14 @@ impl ConfigInfo { "--release-sysroot" => self.sysroot_release_channel = true, "--release" => self.channel = Channel::Release, "--sysroot-panic-abort" => self.sysroot_panic_abort = true, + "--cg_gcc-path" => match args.next() { + Some(arg) if !arg.is_empty() => { + self.cg_gcc_path = Some(arg.into()); + } + _ => { + return Err("Expected a value after `--cg_gcc-path`, found nothing".to_string()) + } + }, _ => return Ok(false), } Ok(true) @@ -159,16 +182,144 @@ impl ConfigInfo { command } + fn download_gccjit_if_needed(&mut self) -> Result<(), String> { + let output_dir = Path::new( + std::env::var("CARGO_TARGET_DIR") + .as_deref() + .unwrap_or("target"), + ) + .join("libgccjit"); + + let commit_hash_file = self.compute_path("libgccjit.version"); + let content = fs::read_to_string(&commit_hash_file).map_err(|_| { + format!( + "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project", + commit_hash_file.display(), + ) + })?; + let commit = content.trim(); + if commit.contains('/') || commit.contains('\\') { + return Err(format!( + "{}: invalid commit hash `{}`", + commit_hash_file.display(), + commit + )); + } + let output_dir = output_dir.join(commit); + if !output_dir.is_dir() { + std::fs::create_dir_all(&output_dir).map_err(|err| { + format!( + "failed to create folder `{}`: {:?}", + output_dir.display(), + err, + ) + })?; + } + let libgccjit_so = output_dir.join("libgccjit.so"); + if !libgccjit_so.is_file() { + // Download time! + let tempfile_name = "libgccjit.so.download"; + let tempfile = output_dir.join(tempfile_name); + let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok(); + + let url = format!( + "https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so", + commit, + ); + + println!("Downloading `{}`...", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut ret = run_command_with_output( + &[ + &"curl", + &"--speed-time", + &"30", + &"--speed-limit", + &"10", // timeout if speed is < 10 bytes/sec for > 30 seconds + &"--connect-timeout", + &"30", // timeout if cannot connect within 30 seconds + &"-o", + &tempfile_name, + &"--retry", + &"3", + &"-SRfL", + if is_in_ci { &"-s" } else { &"--progress-bar" }, + &url.as_str(), + ], + Some(&output_dir), + ); + if ret.is_err() && cfg!(windows) { + eprintln!("Fallback to PowerShell"); + ret = run_command_with_output( + &[ + &"PowerShell.exe", + &"/nologo", + &"-Command", + &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, + tempfile_name, + ).as_str(), + ], + Some(&output_dir), + ); + } + ret?; + + // If we reach this point, it means the file was correctly downloaded, so let's + // rename it! + std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| { + format!( + "Failed to rename `{}` into `{}`: {:?}", + tempfile.display(), + libgccjit_so.display(), + err, + ) + })?; + + println!("Downloaded libgccjit.so version {} successfully!", commit); + } + + self.gcc_path = output_dir + .canonicalize() + .map_err(|err| { + format!( + "Failed to get absolute path of `{}`: {:?}", + output_dir.display(), + err + ) + })? + .display() + .to_string(); + println!("Using `{}` as path for libgccjit", self.gcc_path); + Ok(()) + } + + pub fn compute_path>(&self, other: P) -> PathBuf { + match self.cg_gcc_path { + Some(ref path) => path.join(other), + None => PathBuf::new().join(other), + } + } + pub fn setup_gcc_path(&mut self) -> Result<(), String> { - let ConfigFile { gcc_path, .. } = ConfigFile::new(self.config_file.as_deref())?; + let config_file = self.compute_path(self.config_file.as_deref().unwrap_or("config.toml")); + let ConfigFile { + gcc_path, + download_gccjit, + } = ConfigFile::new(&config_file)?; + if let Some(true) = download_gccjit { + self.download_gccjit_if_needed()?; + return Ok(()); + } self.gcc_path = match gcc_path { Some(path) => path, - // FIXME: Once we support "download", rewrite this. None => { return Err(format!( "missing `gcc-path` value from `{}`", - self.config_file.as_deref().unwrap_or("config.toml"), + config_file.display(), )) } }; @@ -362,7 +513,9 @@ impl ConfigInfo { --release : Build in release mode --release-sysroot : Build sysroot in release mode --sysroot-panic-abort : Build the sysroot without unwinding support - --config-file : Location of the config file to be used" + --config-file : Location of the config file to be used + --cg_gcc-path : Location of the rustc_codegen_gcc root folder (used + for accessing any file from the project)" ); } } diff --git a/libgccjit.version b/libgccjit.version new file mode 100644 index 0000000000000..3fc84f4ddd4ec --- /dev/null +++ b/libgccjit.version @@ -0,0 +1 @@ +2fc8940e1 From 0a4b0af141add015d1388c6b457530fcdd3f0316 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 20:09:54 +0100 Subject: [PATCH 03/10] Generate symbolic link to libgccjit.so as well --- build_system/src/config.rs | 17 +++++++++++++++-- build_system/src/utils.rs | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 0201e3509dcd3..a206bab14a9a8 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,4 +1,6 @@ -use crate::utils::{get_os_name, run_command_with_output, rustc_version_info, split_args}; +use crate::utils::{ + create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args, +}; use std::collections::HashMap; use std::env as std_env; use std::ffi::OsStr; @@ -215,7 +217,8 @@ impl ConfigInfo { ) })?; } - let libgccjit_so = output_dir.join("libgccjit.so"); + let libgccjit_so_name = "libgccjit.so"; + let libgccjit_so = output_dir.join(libgccjit_so_name); if !libgccjit_so.is_file() { // Download time! let tempfile_name = "libgccjit.so.download"; @@ -279,6 +282,16 @@ impl ConfigInfo { })?; println!("Downloaded libgccjit.so version {} successfully!", commit); + create_symlink( + &libgccjit_so.canonicalize().map_err(|err| { + format!( + "Failed to get absolute path of `{}`: {:?}", + libgccjit_so.display(), + err, + ) + })?, + output_dir.join(&format!("{}.0", libgccjit_so_name)), + )?; } self.gcc_path = output_dir diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 046008ae1a247..33dcd9ef7005f 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -374,6 +374,22 @@ pub fn remove_file + ?Sized>(file_path: &P) -> Result<(), String> }) } +pub fn create_symlink, Q: AsRef>(original: P, link: Q) -> Result<(), String> { + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_file; + #[cfg(not(windows))] + let symlink = std::os::unix::fs::symlink; + + symlink(&original, &link).map_err(|err| { + format!( + "failed to create a symlink `{}` to `{}`: {:?}", + original.as_ref().display(), + link.as_ref().display(), + err, + ) + }) +} + #[cfg(test)] mod tests { use super::*; From 5c6cdf5ab6399604695c94031e32087cfe3367ae Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 20:55:14 +0100 Subject: [PATCH 04/10] Add `info` command to help get some information --- build_system/src/config.rs | 35 +++++++++++++++-------------------- build_system/src/info.rs | 19 +++++++++++++++++++ build_system/src/main.rs | 5 +++++ 3 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 build_system/src/info.rs diff --git a/build_system/src/config.rs b/build_system/src/config.rs index a206bab14a9a8..48be515a1f0f8 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -121,6 +121,9 @@ pub struct ConfigInfo { pub gcc_path: String, config_file: Option, cg_gcc_path: Option, + // Needed for the `info` command which doesn't want to actually download the lib if needed, + // just to set the `gcc_path` field to display it. + pub no_download: bool, } impl ConfigInfo { @@ -204,7 +207,7 @@ impl ConfigInfo { return Err(format!( "{}: invalid commit hash `{}`", commit_hash_file.display(), - commit + commit, )); } let output_dir = output_dir.join(commit); @@ -217,9 +220,17 @@ impl ConfigInfo { ) })?; } + let output_dir = output_dir.canonicalize().map_err(|err| { + format!( + "Failed to get absolute path of `{}`: {:?}", + output_dir.display(), + err + ) + })?; + let libgccjit_so_name = "libgccjit.so"; let libgccjit_so = output_dir.join(libgccjit_so_name); - if !libgccjit_so.is_file() { + if !libgccjit_so.is_file() && !self.no_download { // Download time! let tempfile_name = "libgccjit.so.download"; let tempfile = output_dir.join(tempfile_name); @@ -283,28 +294,12 @@ impl ConfigInfo { println!("Downloaded libgccjit.so version {} successfully!", commit); create_symlink( - &libgccjit_so.canonicalize().map_err(|err| { - format!( - "Failed to get absolute path of `{}`: {:?}", - libgccjit_so.display(), - err, - ) - })?, + &libgccjit_so, output_dir.join(&format!("{}.0", libgccjit_so_name)), )?; } - self.gcc_path = output_dir - .canonicalize() - .map_err(|err| { - format!( - "Failed to get absolute path of `{}`: {:?}", - output_dir.display(), - err - ) - })? - .display() - .to_string(); + self.gcc_path = output_dir.display().to_string(); println!("Using `{}` as path for libgccjit", self.gcc_path); Ok(()) } diff --git a/build_system/src/info.rs b/build_system/src/info.rs new file mode 100644 index 0000000000000..ea38791d38c9a --- /dev/null +++ b/build_system/src/info.rs @@ -0,0 +1,19 @@ +use crate::config::ConfigInfo; + +pub fn run() -> Result<(), String> { + let mut config = ConfigInfo::default(); + + // We skip binary name and the `info` command. + let mut args = std::env::args().skip(2); + while let Some(arg) = args.next() { + if arg == "--help" { + println!("Display the path where the libgccjit will be located"); + return Ok(()); + } + config.parse_argument(&arg, &mut args)?; + } + config.no_download = true; + config.setup_gcc_path()?; + println!("{}", config.gcc_path); + Ok(()) +} diff --git a/build_system/src/main.rs b/build_system/src/main.rs index 102c5486a75e4..c6958f0c5124e 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -5,6 +5,7 @@ mod build; mod cargo; mod clean; mod config; +mod info; mod prepare; mod rustc_info; mod test; @@ -29,6 +30,7 @@ Available commands for build_system: prepare : Run prepare command build : Run build command test : Run test command + info: : Run info command --help : Show this message" ); } @@ -39,6 +41,7 @@ pub enum Command { Prepare, Build, Test, + Info, } fn main() { @@ -52,6 +55,7 @@ fn main() { Some("prepare") => Command::Prepare, Some("build") => Command::Build, Some("test") => Command::Test, + Some("info") => Command::Info, Some("--help") => { usage(); process::exit(0); @@ -70,6 +74,7 @@ fn main() { Command::Prepare => prepare::run(), Command::Build => build::run(), Command::Test => test::run(), + Command::Info => info::run(), } { eprintln!("Command failed to run: {e}"); process::exit(1); From 65f4b6354d8899e9292f8666d2804a0306b0770e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 18:11:37 +0100 Subject: [PATCH 05/10] Add CI for download config --- .github/workflows/download.yml | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/download.yml diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml new file mode 100644 index 0000000000000..86a8459a33c91 --- /dev/null +++ b/.github/workflows/download.yml @@ -0,0 +1,97 @@ +name: Check download command + +on: + - push + - pull_request + +permissions: + contents: read + +env: + # Enable backtraces for easier debugging + RUST_BACKTRACE: 1 + +jobs: + build: + runs-on: ubuntu-22.04 + + strategy: + fail-fast: false + matrix: + commands: [ + "--mini-tests", + "--std-tests", + # FIXME: re-enable asm tests when GCC can emit in the right syntax. + # "--asm-tests", + "--test-libcore", + "--extended-rand-tests", + "--extended-regex-example-tests", + "--extended-regex-tests", + "--test-successful-rustc --nb-parts 2 --current-part 0", + "--test-successful-rustc --nb-parts 2 --current-part 1", + "--projects", + ] + + steps: + - uses: actions/checkout@v3 + + # `rustup show` installs from rust-toolchain.toml + - name: Setup rust toolchain + run: rustup show + + - name: Setup rust cache + uses: Swatinem/rust-cache@v2 + + - name: Install packages + # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests. + run: sudo apt-get install ninja-build ripgrep llvm-14-tools + + - name: Setup path to libgccjit + run: | + echo 'download-gccjit = true' > config.toml + + - name: Set env + run: | + echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV + + #- name: Cache rust repository + ## We only clone the rust repository for rustc tests + #if: ${{ contains(matrix.commands, 'rustc') }} + #uses: actions/cache@v3 + #id: cache-rust-repository + #with: + #path: rust + #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }} + + - name: Build + run: | + ./y.sh prepare --only-libcore + # TODO: remove --features master when it is back to the default. + ./y.sh build --features master + # TODO: remove --features master when it is back to the default. + + - name: Set env (part 2) + run: | + # Set the `LD_LIBRARY_PATH` and `LIBRARY_PATH` env variables... + echo "LD_LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV + echo "LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV + + - name: Build (part 2) + run: | + echo "LIBRARY_PATH=" $LIBRARY_PATH + cargo test --features master + ./y.sh clean all + + - name: Prepare dependencies + run: | + git config --global user.email "user@example.com" + git config --global user.name "User" + ./y.sh prepare + + - name: Add more failing tests because the sysroot is not compiled with LTO + run: cat tests/failing-non-lto-tests.txt >> tests/failing-ui-tests.txt + + - name: Run tests + run: | + # TODO: remove --features master when it is back to the default. + ./y.sh test --features master --release --clean --build-sysroot ${{ matrix.commands }} From d04ffb0ffc10bbf2623f32fc6840ed3a7614ff33 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 12 Feb 2024 22:52:57 +0100 Subject: [PATCH 06/10] Update lang_tests_common.rs test --- tests/lang_tests_common.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/lang_tests_common.rs b/tests/lang_tests_common.rs index 029a3b98ff23a..33dc6ef62ae52 100644 --- a/tests/lang_tests_common.rs +++ b/tests/lang_tests_common.rs @@ -21,11 +21,16 @@ pub fn main_inner(profile: Profile) { let tempdir = TempDir::new().expect("temp dir"); let current_dir = current_dir().expect("current dir"); let current_dir = current_dir.to_str().expect("current dir").to_string(); - let gcc_path = Toml::parse(include_str!("../config.toml")) - .expect("Failed to parse `config.toml`") - .get_string("gcc-path") - .expect("Missing `gcc-path` key in `config.toml`") - .to_string(); + let toml = Toml::parse(include_str!("../config.toml")) + .expect("Failed to parse `config.toml`"); + let gcc_path = if let Ok(gcc_path) = toml.get_string("gcc-path") { + PathBuf::from(gcc_path.to_string()) + } else { + // then we try to retrieve it from the `target` folder. + let commit = include_str!("../libgccjit.version").trim(); + Path::new("target/libgccjit").join(commit) + }; + let gcc_path = Path::new(&gcc_path) .canonicalize() .expect("failed to get absolute path of `gcc-path`") From 59546ea2d53c9f3c9f0e87bb2852d6b972ec8f8b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 14:20:39 +0100 Subject: [PATCH 07/10] Merge `download.yml` into `ci.yml` --- .github/workflows/ci.yml | 20 +++---- .github/workflows/download.yml | 97 ---------------------------------- 2 files changed, 10 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/download.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba64f40acc4ba..e4678c4e2af4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,19 +49,10 @@ jobs: # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests. run: sudo apt-get install ninja-build ripgrep llvm-14-tools - - name: Download artifact - run: curl -LO https://github.com/antoyo/gcc/releases/latest/download/gcc-13.deb - - - name: Setup path to libgccjit - run: | - sudo dpkg --force-overwrite -i gcc-13.deb - echo 'gcc-path = "/usr/lib/"' > config.toml - - name: Set env run: | echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV - echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV + echo 'download-gccjit = true' > config.toml #- name: Cache rust repository ## We only clone the rust repository for rustc tests @@ -78,6 +69,15 @@ jobs: # TODO: remove --features master when it is back to the default. ./y.sh build --features master # TODO: remove --features master when it is back to the default. + + - name: Set env (part 2) + run: | + # Set the `LD_LIBRARY_PATH` and `LIBRARY_PATH` env variables... + echo "LD_LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV + echo "LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV + + - name: Build (part 2) + run: | cargo test --features master ./y.sh clean all diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml deleted file mode 100644 index 86a8459a33c91..0000000000000 --- a/.github/workflows/download.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Check download command - -on: - - push - - pull_request - -permissions: - contents: read - -env: - # Enable backtraces for easier debugging - RUST_BACKTRACE: 1 - -jobs: - build: - runs-on: ubuntu-22.04 - - strategy: - fail-fast: false - matrix: - commands: [ - "--mini-tests", - "--std-tests", - # FIXME: re-enable asm tests when GCC can emit in the right syntax. - # "--asm-tests", - "--test-libcore", - "--extended-rand-tests", - "--extended-regex-example-tests", - "--extended-regex-tests", - "--test-successful-rustc --nb-parts 2 --current-part 0", - "--test-successful-rustc --nb-parts 2 --current-part 1", - "--projects", - ] - - steps: - - uses: actions/checkout@v3 - - # `rustup show` installs from rust-toolchain.toml - - name: Setup rust toolchain - run: rustup show - - - name: Setup rust cache - uses: Swatinem/rust-cache@v2 - - - name: Install packages - # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests. - run: sudo apt-get install ninja-build ripgrep llvm-14-tools - - - name: Setup path to libgccjit - run: | - echo 'download-gccjit = true' > config.toml - - - name: Set env - run: | - echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV - - #- name: Cache rust repository - ## We only clone the rust repository for rustc tests - #if: ${{ contains(matrix.commands, 'rustc') }} - #uses: actions/cache@v3 - #id: cache-rust-repository - #with: - #path: rust - #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }} - - - name: Build - run: | - ./y.sh prepare --only-libcore - # TODO: remove --features master when it is back to the default. - ./y.sh build --features master - # TODO: remove --features master when it is back to the default. - - - name: Set env (part 2) - run: | - # Set the `LD_LIBRARY_PATH` and `LIBRARY_PATH` env variables... - echo "LD_LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV - echo "LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV - - - name: Build (part 2) - run: | - echo "LIBRARY_PATH=" $LIBRARY_PATH - cargo test --features master - ./y.sh clean all - - - name: Prepare dependencies - run: | - git config --global user.email "user@example.com" - git config --global user.name "User" - ./y.sh prepare - - - name: Add more failing tests because the sysroot is not compiled with LTO - run: cat tests/failing-non-lto-tests.txt >> tests/failing-ui-tests.txt - - - name: Run tests - run: | - # TODO: remove --features master when it is back to the default. - ./y.sh test --features master --release --clean --build-sysroot ${{ matrix.commands }} From 2bcc73540cbadaae43b539567252f79dffd43f5a Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 14:44:19 +0100 Subject: [PATCH 08/10] Don't join config file path if provided through `--config-file` option --- build_system/src/config.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 48be515a1f0f8..b48e132ebd2aa 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -312,7 +312,10 @@ impl ConfigInfo { } pub fn setup_gcc_path(&mut self) -> Result<(), String> { - let config_file = self.compute_path(self.config_file.as_deref().unwrap_or("config.toml")); + let config_file = match self.config_file.as_deref() { + Some(config_file) => config_file.into(), + None => self.compute_path("config.toml"), + }; let ConfigFile { gcc_path, download_gccjit, From 1096b1b8db1ac3bdb13f77e3726a00d9516af5a0 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 15:01:13 +0100 Subject: [PATCH 09/10] Add more explanation on what `cg_gcc_path` is used for and improve help message for `--cg_gcc-path` --- build_system/src/config.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index b48e132ebd2aa..fc2ef7b797d56 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -120,6 +120,9 @@ pub struct ConfigInfo { pub sysroot_path: String, pub gcc_path: String, config_file: Option, + // This is used in particular in rust compiler bootstrap because it doesn't run at the root + // of the `cg_gcc` folder, making it complicated for us to get access to local files we need + // like `libgccjit.version` or `config.toml`. cg_gcc_path: Option, // Needed for the `info` command which doesn't want to actually download the lib if needed, // just to set the `gcc_path` field to display it. @@ -526,7 +529,7 @@ impl ConfigInfo { --sysroot-panic-abort : Build the sysroot without unwinding support --config-file : Location of the config file to be used --cg_gcc-path : Location of the rustc_codegen_gcc root folder (used - for accessing any file from the project)" + when ran from another directory)" ); } } From b80a99922be9866d4117931e50127f36010a8dc9 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 15:43:07 +0100 Subject: [PATCH 10/10] Improve code readability and add more code comments --- build_system/src/config.rs | 146 ++++++++++++++++++++----------------- 1 file changed, 80 insertions(+), 66 deletions(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index fc2ef7b797d56..475f9b300f74c 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -190,6 +190,83 @@ impl ConfigInfo { command } + fn download_gccjit( + &self, + output_dir: &Path, + libgccjit_so_name: &str, + commit: &str, + ) -> Result<(), String> { + // Download time! + let tempfile_name = format!("{}.download", libgccjit_so_name); + let tempfile = output_dir.join(&tempfile_name); + let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok(); + + let url = format!( + "https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so", + commit, + ); + + println!("Downloading `{}`...", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut ret = run_command_with_output( + &[ + &"curl", + &"--speed-time", + &"30", + &"--speed-limit", + &"10", // timeout if speed is < 10 bytes/sec for > 30 seconds + &"--connect-timeout", + &"30", // timeout if cannot connect within 30 seconds + &"-o", + &tempfile_name, + &"--retry", + &"3", + &"-SRfL", + if is_in_ci { &"-s" } else { &"--progress-bar" }, + &url.as_str(), + ], + Some(&output_dir), + ); + if ret.is_err() && cfg!(windows) { + eprintln!("Fallback to PowerShell"); + ret = run_command_with_output( + &[ + &"PowerShell.exe", + &"/nologo", + &"-Command", + &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, + tempfile_name, + ).as_str(), + ], + Some(&output_dir), + ); + } + ret?; + + let libgccjit_so = output_dir.join(libgccjit_so_name); + // If we reach this point, it means the file was correctly downloaded, so let's + // rename it! + std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| { + format!( + "Failed to rename `{}` into `{}`: {:?}", + tempfile.display(), + libgccjit_so.display(), + err, + ) + })?; + + println!("Downloaded libgccjit.so version {} successfully!", commit); + // We need to create a link named `libgccjit.so.0` because that's what the linker is + // looking for. + create_symlink( + &libgccjit_so, + output_dir.join(&format!("{}.0", libgccjit_so_name)), + ) + } + fn download_gccjit_if_needed(&mut self) -> Result<(), String> { let output_dir = Path::new( std::env::var("CARGO_TARGET_DIR") @@ -206,6 +283,8 @@ impl ConfigInfo { ) })?; let commit = content.trim(); + // This is a very simple check to ensure this is not a path. For the rest, it'll just fail + // when trying to download the file so we should be fine. if commit.contains('/') || commit.contains('\\') { return Err(format!( "{}: invalid commit hash `{}`", @@ -234,72 +313,7 @@ impl ConfigInfo { let libgccjit_so_name = "libgccjit.so"; let libgccjit_so = output_dir.join(libgccjit_so_name); if !libgccjit_so.is_file() && !self.no_download { - // Download time! - let tempfile_name = "libgccjit.so.download"; - let tempfile = output_dir.join(tempfile_name); - let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok(); - - let url = format!( - "https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so", - commit, - ); - - println!("Downloading `{}`...", url); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - let mut ret = run_command_with_output( - &[ - &"curl", - &"--speed-time", - &"30", - &"--speed-limit", - &"10", // timeout if speed is < 10 bytes/sec for > 30 seconds - &"--connect-timeout", - &"30", // timeout if cannot connect within 30 seconds - &"-o", - &tempfile_name, - &"--retry", - &"3", - &"-SRfL", - if is_in_ci { &"-s" } else { &"--progress-bar" }, - &url.as_str(), - ], - Some(&output_dir), - ); - if ret.is_err() && cfg!(windows) { - eprintln!("Fallback to PowerShell"); - ret = run_command_with_output( - &[ - &"PowerShell.exe", - &"/nologo", - &"-Command", - &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, - tempfile_name, - ).as_str(), - ], - Some(&output_dir), - ); - } - ret?; - - // If we reach this point, it means the file was correctly downloaded, so let's - // rename it! - std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| { - format!( - "Failed to rename `{}` into `{}`: {:?}", - tempfile.display(), - libgccjit_so.display(), - err, - ) - })?; - - println!("Downloaded libgccjit.so version {} successfully!", commit); - create_symlink( - &libgccjit_so, - output_dir.join(&format!("{}.0", libgccjit_so_name)), - )?; + self.download_gccjit(&output_dir, libgccjit_so_name, commit)?; } self.gcc_path = output_dir.display().to_string();