From fe2fb7522bde19e5bdf54c8abe73f9c375c566a0 Mon Sep 17 00:00:00 2001 From: 33KK Date: Thu, 26 Oct 2023 09:40:39 +0200 Subject: [PATCH] Move the building to python --- build.rs | 216 ++++++-------------------------------------- scripts/build_v8.py | 174 +++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 189 deletions(-) create mode 100644 scripts/build_v8.py diff --git a/build.rs b/build.rs index 7e57c1861f..41d63e090f 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; use std::env; use std::env::consts; -use std::ffi::{OsStr, OsString}; +use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::process::Command; use which::which; @@ -18,16 +17,6 @@ macro_rules! system { .success()); }; } -macro_rules! s { - ($expr:expr) => { - $expr.to_string() - }; -} -macro_rules! q { - ($expr:expr) => { - format!("\"{}\"", $expr) - }; -} fn main() { set_cargo_rerun(); @@ -58,11 +47,7 @@ fn main() { return; } - build_v8( - crate_dir, - out_dir, - gn_out_dir.as_os_str().try_into().unwrap(), - ); + build_v8(crate_dir, out_dir, gn_out_dir); } fn gn_os<'a>(os: &'a str) -> &'a str { @@ -84,7 +69,7 @@ fn gn_arch<'a>(arch: &'a str) -> &'a str { } } -fn build_v8(crate_dir: PathBuf, out_dir: PathBuf, gn_out_dir: &str) { +fn build_v8(crate_dir: PathBuf, out_dir: PathBuf, gn_out_dir: PathBuf) { env::set_var("DEPOT_TOOLS_WIN_TOOLCHAIN", "0"); // google uses internal visual studio by default env::set_var("PYTHONDONTWRITEBYTECODE", "1"); // disable __pycache__ @@ -95,52 +80,7 @@ fn build_v8(crate_dir: PathBuf, out_dir: PathBuf, gn_out_dir: &str) { let gn_target_os = gn_os(&target_os); let gn_target_arch = gn_arch(&target_arch); - let mut gn_args = HashMap::<&str, String>::from([ - ("clang_use_chrome_plugins", s!(false)), - ("is_component_build", s!(false)), - ("linux_use_bundled_binutils", s!(false)), - ("use_dummy_lastchange", s!(true)), - ("use_sysroot", s!(false)), - ("win_crt_flavor_agnostic", s!(true)), - // Minimize size of debuginfo in distributed static library. - ("line_tables_only", s!(true)), - ("no_inline_line_tables", s!(true)), - ("symbol_level", s!(1)), - ("use_debug_fission", s!(false)), - ("v8_enable_sandbox", s!(false)), - ("v8_enable_snapshot_compression", s!(false)), - ("v8_enable_javascript_promise_hooks", s!(true)), - ("v8_promise_internal_field_count", s!(1)), - ("v8_use_external_startup_data", s!(false)), - ("v8_use_snapshot", s!(true)), - // Disable handle zapping for performance - ("v8_enable_handle_zapping", s!(false)), - // Ensure allocation of typed arrays and arraybuffers always goes through - // the embedder's ArrayBufferAllocator, otherwise small buffers get moved - // around by the garbage collector but embedders normally want them to have - // fixed addresses. - ("v8_typed_array_max_size_in_heap", s!(0)), - // Enabling the shared read-only heap comes with a restriction that all - // isolates running at the same time must be created from the same snapshot. - // This is problematic for Deno, which has separate "runtime" and "typescript - // compiler" snapshots, and sometimes uses them both at the same time. - ("v8_enable_shared_ro_heap", s!(false)), - // V8 11.6 hardcoded an assumption in `mksnapshot` that shared RO heap - // is enabled. In our case it's disabled so without this flag we can't - // compile. - ("v8_enable_verify_heap", s!(false)), - // V8 introduced a bug in 11.1 that causes the External Pointer Table to never - // be cleaned which causes resource exhaustion. Disabling pointer compression - // makes sure that the EPT is not used. - // https://bugs.chromium.org/p/v8/issues/detail?id=13640&q=garbage%20collection&can=2 - ("v8_enable_pointer_compression", s!(false)), - // Maglev *should* be supported when pointer compression is disabled as per - // https://chromium-review.googlesource.com/c/v8/v8/+/4753150, but it still - // fails to compile. - ("v8_enable_maglev", s!(false)), - ]); - - let is_debug = if target_os == "windows" { + let is_debug = if cfg!(target_os = "windows") { false } else { match env::var("PROFILE").unwrap().as_str() { @@ -149,90 +89,36 @@ fn build_v8(crate_dir: PathBuf, out_dir: PathBuf, gn_out_dir: &str) { } }; - #[cfg(not(feature = "use_custom_libcxx"))] - gn_args.insert("use_custom_libcxx", s!(false)); - - gn_args.insert("is_debug", s!(is_debug)); - gn_args.insert("target_cpu", q!(gn_target_arch)); - gn_args.insert("v8_target_cpu", q!(gn_target_arch)); - - if let Some(cc_wrapper) = find_cc_wrapper() { - gn_args.insert("cc_wrapper", q!(cc_wrapper.to_string_lossy())); - } - - if target_os != consts::OS { - gn_args.insert("target_os", q!(gn_target_os)); - } - - let use_sysroot = target_arch != consts::ARCH; - gn_args.insert("use_sysroot", s!(use_sysroot)); - - if env::var_os("DISABLE_CLANG").is_some() { - gn_args.insert("is_clang", s!(false)); - gn_args.insert("line_tables_only", s!(false)); - } else if let Ok(clang_base_path) = env::var("CLANG_BASE_PATH") { - gn_args.insert("clang_base_path", q!(clang_base_path)); - gn_args.insert("treat_warnings_as_errors", s!(false)); - } - - let extra_gn_args = env::var("GN_ARGS").unwrap_or_default(); - - for pair in extra_gn_args - .split_whitespace() - .map(|pair| pair.split_once("=")) - { - if let Some((k, v)) = pair { - gn_args.insert(k, v.to_string()); - } - } - - let gn_args = gn_args - .iter() - .map(|(key, value)| format!("{key}={value}")) - .collect::>() - .join(" "); - - env::set_var("GN_ARGS", &gn_args); - - println!("[*] crate_dir : {}", crate_dir.display()); - println!("[*] out_dir : {}", out_dir.display()); - println!("[*] gn_out_dir : {}", gn_out_dir); - println!("[*] gn_args : {}", gn_args); - - env::set_current_dir(&out_dir).unwrap(); + let use_custom_libcxx = cfg!(feature = "use_custom_libcxx"); let python = find_python().expect("Can't find python"); system!( &python, [ - crate_dir.join("scripts/download_v8.py").as_os_str(), - crate_dir.as_os_str(), - out_dir.as_os_str(), - OsStr::new(gn_host_os), - OsStr::new(gn_host_arch), - OsStr::new(gn_target_os), - OsStr::new(gn_target_arch), - OsStr::new(&format!("--host-os={gn_host_os}")), - OsStr::new(&format!("--host-cpu={gn_host_arch}")), + crate_dir + .join("scripts/build_v8.py") + .to_string_lossy() + .as_ref(), + "--crate-root", + crate_dir.to_string_lossy().as_ref(), + "--root", + out_dir.to_string_lossy().as_ref(), + "--gn-root", + gn_out_dir.to_string_lossy().as_ref(), + "--host-os", + gn_host_os, + "--host-cpu", + gn_host_arch, + "--target-os", + gn_target_os, + "--target-cpu", + gn_target_arch, + "--is-debug", + is_debug.to_string().as_ref(), + "--use-custom-libcxx", + use_custom_libcxx.to_string().as_ref(), ] ); - - let gn = find_gn(&out_dir).expect("Can't find gn"); - let gn_se = format!("--script-executable={}", python.to_string_lossy()); - - system!( - &gn, - [&gn_se, "gen", gn_out_dir, &format!("--args={gn_args}")] - ); - - if env::var_os("PRINT_GN_ARGS").is_some() { - system!(&gn, [&gn_se, "args", gn_out_dir, "--list"]); - } - - system!( - find_ninja(&out_dir).expect("Can't find ninja"), - ["-C", gn_out_dir, "rusty_v8"] - ); } fn set_cargo_rerun() { @@ -332,51 +218,3 @@ fn find_python() -> Option { return None; }) } - -fn find_ninja(out_dir: &Path) -> Option { - let mut ninja_path = out_dir.join("third_party").join("ninja").join("ninja"); - ninja_path.set_extension(consts::EXE_EXTENSION); - - Some(if let Some(path) = env::var_os("NINJA") { - path - } else if ninja_path.exists() { - ninja_path.into() - } else if let Ok(path) = which("ninja") { - path.into() - } else { - return None; - }) -} - -fn find_gn(out_dir: &Path) -> Option { - let platform = match consts::OS { - "linux" => "linux64", - "windows" => "win", - "macos" => "mac", - _ => "unsupported", - }; - let mut gn_path = out_dir.join("buildtools").join(platform).join("gn"); - gn_path.set_extension(consts::EXE_EXTENSION); - - Some(if let Some(path) = env::var_os("GN") { - path - } else if gn_path.exists() { - gn_path.into() - } else { - return None; - }) -} - -fn find_cc_wrapper() -> Option { - Some(if let Some(path) = env::var_os("SCCACHE") { - path - } else if let Ok(path) = which("sccache") { - path.into() - } else if let Some(path) = env::var_os("CCACHE") { - path - } else if let Ok(path) = which("ccache") { - path.into() - } else { - return None; - }) -} diff --git a/scripts/build_v8.py b/scripts/build_v8.py new file mode 100644 index 0000000000..b6b13c284f --- /dev/null +++ b/scripts/build_v8.py @@ -0,0 +1,174 @@ +import os +import sys +import shutil +import argparse +import download_v8 +from common import system + + +def default_args(): + return { + "clang_use_chrome_plugins": False, + "is_component_build": False, + "linux_use_bundled_binutils": False, + "use_dummy_lastchange": True, + "use_sysroot": False, + "win_crt_flavor_agnostic": True, + # Minimize size of debuginfo in distributed static library. + "line_tables_only": True, + "no_inline_line_tables": True, + "symbol_level": 1, + "use_debug_fission": False, + "v8_enable_sandbox": False, + "v8_enable_snapshot_compression": False, + "v8_enable_javascript_promise_hooks": True, + "v8_promise_internal_field_count": 1, + "v8_use_external_startup_data": False, + "v8_use_snapshot": True, + # Disable handle zapping for performance + "v8_enable_handle_zapping": False, + # Ensure allocation of typed arrays and arraybuffers always goes through + # the embedder's ArrayBufferAllocator, otherwise small buffers get moved + # around by the garbage collector but embedders normally want them to have + # fixed addresses. + "v8_typed_array_max_size_in_heap": 0, + # Enabling the shared read-only heap comes with a restriction that all + # isolates running at the same time must be created from the same snapshot. + # This is problematic for Deno, which has separate "runtime" and "typescript + # compiler" snapshots, and sometimes uses them both at the same time. + "v8_enable_shared_ro_heap": False, + # V8 11.6 hardcoded an assumption in `mksnapshot` that shared RO heap + # is enabled. In our case it's disabled so without this flag we can't + # compile. + "v8_enable_verify_heap": False, + # V8 introduced a bug in 11.1 that causes the External Pointer Table to never + # be cleaned which causes resource exhaustion. Disabling pointer compression + # makes sure that the EPT is not used. + # https:#bugs.chromium.org/p/v8/issues/detail?id=13640&q=garbage%20collection&can=2 + "v8_enable_pointer_compression": False, + # Maglev *should* be supported when pointer compression is disabled as per + # https:#chromium-review.googlesource.com/c/v8/v8/+/4753150, but it still + # fails to compile. + "v8_enable_maglev": False, + } + + +def find_cc_wrapper(): + return ( + os.environ.get("SCCACHE") + or shutil.which("sccache") + or os.environ.get("CCACHE") + or shutil.which("ccache") + ) + + +def find_ninja(root): + return ( + os.environ.get("NINJA") + or shutil.which(os.path.join(root, "third_party/ninja/ninja")) + or shutil.which("ninja") + ) + + +def find_gn(root, host_os): + if host_os == "linux": + host_os = "linux64" + return os.environ.get("GN") or shutil.which( + os.path.join(root, "buildtools", host_os, "gn") + ) + + +def s(value): + if type(value) is bool: + return str(value).lower() + else: + return str(value) + + +def q(value): + return f'"{value}"' + + +def build_v8( + crate_root, + root, + gn_root, + host_os, + host_cpu, + target_os, + target_cpu, + is_debug, + use_custom_libcxx, +): + gn_args = default_args() + gn_args["is_debug"] = is_debug + gn_args["target_cpu"] = q(target_cpu) + gn_args["v8_target_cpu"] = q(target_cpu) + + gn_args["cc_wrapper"] = q(find_cc_wrapper()) + + if not use_custom_libcxx: + gn_args["use_custom_libcxx"] = False + + if host_cpu != target_cpu: + gn_args["use_sysroot"] = True + + if host_os != target_os: + gn_args["target_os"] = q(target_os) + + if "DISABLE_CLANG" in os.environ: + gn_args["is_clang"] = False + gn_args["line_tables_only"] = False + elif "CLANG_BASE_PATH" in os.environ: + gn_args["clang_base_path"] = q(os.environ["CLANG_BASE_PATH"]) + gn_args["clang_use_chrome_plugins"] = False + gn_args["treat_warnings_as_errors"] = False + + for arg in (os.environ.get("GN_ARGS") or "").split(): + k, v = arg.split("=", 1) + gn_args[k] = v + + gn_args = [f"{k}={s(v)}" for k, v in gn_args.items()] + gn_args = " ".join(gn_args) + print(gn_args) + + download_v8.main( + crate_root, + root, + [host_os, host_cpu, target_os, target_cpu], + host_os=host_os, + host_cpu=host_cpu, + ) + + gn_se = f"--script-executable={sys.executable}" + gn_r = f"--root={root}" + system([find_gn(root, host_os), gn_r, gn_se, "gen", gn_root, f"--args={gn_args}"]) + if "PRINT_GN_ARGS" in os.environ: + system([find_gn(root, host_os), gn_r, gn_se, "args", gn_root, "--list"]) + + system([find_ninja(root), "-C", gn_root, "rusty_v8"]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--crate-root", required=True) + parser.add_argument("--root", required=True) + parser.add_argument("--gn-root", required=True) + parser.add_argument("--host-os", required=True) + parser.add_argument("--host-cpu", required=True) + parser.add_argument("--target-os", required=True) + parser.add_argument("--target-cpu", required=True) + parser.add_argument("--is-debug", type=bool, default=False) + parser.add_argument("--use-custom-libcxx", type=bool, default=True) + args = parser.parse_args() + build_v8( + args.crate_root, + args.root, + args.gn_root, + args.host_os, + args.host_cpu, + args.target_os, + args.target_cpu, + args.is_debug, + args.use_custom_libcxx, + )