From a4ca4d8560220d4f6f1c1e8ebf716897a63ec4e5 Mon Sep 17 00:00:00 2001 From: Gohlub <62673775+Gohlub@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:46:52 -0400 Subject: [PATCH 1/4] added support for building core packages in parallel with ryon --- README.md | 3 ++ scripts/build-packages/Cargo.toml | 1 + scripts/build-packages/src/main.rs | 50 ++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5be3e144d..00b4b94ae 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,12 @@ rustup target add wasm32-wasip1 # The compiled packages will be at `hyperdrive/target/packages.zip`. # The compiled binary will be at `hyperdrive/target/debug/hyperdrive`. # OPTIONAL: --release flag (slower build; faster runtime; binary at `hyperdrive/target/release/hyperdrive`). +# OPTIONAL: --parallel flag to build packages in parallel (faster but uses more resources). cd hyperdrive cargo run -p build-packages +# OR for parallel builds: +# cargo run -p build-packages -- --parallel # OPTIONAL: --release flag cargo build -p hyperdrive ``` diff --git a/scripts/build-packages/Cargo.toml b/scripts/build-packages/Cargo.toml index 6eb721a09..a6c822e65 100644 --- a/scripts/build-packages/Cargo.toml +++ b/scripts/build-packages/Cargo.toml @@ -8,6 +8,7 @@ anyhow = "1.0.71" clap = "4" fs-err = "2.11" kit = { git = "https://github.com/hyperware-ai/kit", rev = "275f02c" } +rayon = "1.8" serde = "1" serde_json = "1" tokio = "1.28" diff --git a/scripts/build-packages/src/main.rs b/scripts/build-packages/src/main.rs index ce8a9acb9..0f2f696e6 100644 --- a/scripts/build-packages/src/main.rs +++ b/scripts/build-packages/src/main.rs @@ -6,6 +6,7 @@ use std::{ use clap::{Arg, Command}; use fs_err as fs; +use rayon::prelude::*; use zip::write::FileOptions; #[derive(serde::Deserialize, serde::Serialize)] @@ -113,6 +114,12 @@ fn main() -> anyhow::Result<()> { .help("Set output filename (default: packages-{features}.zip)") .action(clap::ArgAction::Set), ) + .arg( + Arg::new("PARALLEL") + .long("parallel") + .help("Build packages in parallel (faster but uses more resources)") + .action(clap::ArgAction::SetTrue), + ) .get_matches(); // hyperdrive/target/debug/build-package @@ -137,6 +144,7 @@ fn main() -> anyhow::Result<()> { let features = features.join(","); let skip_frontend = matches.get_flag("SKIP_FRONTEND"); + let parallel = matches.get_flag("PARALLEL"); let build_parameters = fs::read(hyperdrive_dir.join("packages-build-parameters.json"))?; let build_parameters: HashMap = @@ -157,7 +165,8 @@ fn main() -> anyhow::Result<()> { }) .collect(); - let results: Vec)>> = fs::read_dir(&packages_dir)? + // First, collect all package info sequentially (since we're mutating build_parameters) + let package_build_info: Vec<(PathBuf, String, String, Vec, bool)> = fs::read_dir(&packages_dir)? .filter_map(|entry| { let entry_path = match entry { Ok(e) => e.path(), @@ -201,17 +210,46 @@ fn main() -> anyhow::Result<()> { } else { format!("{features},{}", package_specific_features.join(",")) }; - Some(build_and_zip_package( - entry_path.clone(), - child_pkg_path.to_str().unwrap(), - skip_frontend, - &package_specific_features, + Some(( + entry_path, + child_pkg_path.to_string_lossy().to_string(), + package_specific_features, local_dependency_array, is_hyperapp, )) }) .collect(); + let results: Vec)>> = if parallel { + package_build_info + .into_par_iter() + .map(|(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { + build_and_zip_package( + entry_path, + &child_pkg_path, + skip_frontend, + &package_features, + local_deps, + is_hyperapp, + ) + }) + .collect() + } else { + package_build_info + .into_iter() + .map(|(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { + build_and_zip_package( + entry_path, + &child_pkg_path, + skip_frontend, + &package_features, + local_deps, + is_hyperapp, + ) + }) + .collect() + }; + let mut file_to_metadata = std::collections::HashMap::new(); let target_dir = top_level_dir.join("target"); From 46764fd247d994bcc75b5ea8e11c11ab9650ed4e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:47:22 +0000 Subject: [PATCH 2/4] Format Rust code using rustfmt --- scripts/build-packages/src/main.rs | 141 +++++++++++++++-------------- 1 file changed, 73 insertions(+), 68 deletions(-) diff --git a/scripts/build-packages/src/main.rs b/scripts/build-packages/src/main.rs index 0f2f696e6..052c33916 100644 --- a/scripts/build-packages/src/main.rs +++ b/scripts/build-packages/src/main.rs @@ -166,87 +166,92 @@ fn main() -> anyhow::Result<()> { .collect(); // First, collect all package info sequentially (since we're mutating build_parameters) - let package_build_info: Vec<(PathBuf, String, String, Vec, bool)> = fs::read_dir(&packages_dir)? - .filter_map(|entry| { - let entry_path = match entry { - Ok(e) => e.path(), - Err(_) => return None, - }; - let child_pkg_path = entry_path.join("pkg"); - if !child_pkg_path.exists() { - // don't run on, e.g., `.DS_Store` - return None; - } - let (local_dependency_array, is_hyperapp, package_specific_features) = - if let Some(filename) = entry_path.file_name() { - if let Some(maybe_params) = - build_parameters.remove(&filename.to_string_lossy().to_string()) - { - ( - maybe_params.local_dependencies.unwrap_or_default(), - maybe_params.is_hyperapp.unwrap_or_default(), - maybe_params.features.unwrap_or_default(), - ) + let package_build_info: Vec<(PathBuf, String, String, Vec, bool)> = + fs::read_dir(&packages_dir)? + .filter_map(|entry| { + let entry_path = match entry { + Ok(e) => e.path(), + Err(_) => return None, + }; + let child_pkg_path = entry_path.join("pkg"); + if !child_pkg_path.exists() { + // don't run on, e.g., `.DS_Store` + return None; + } + let (local_dependency_array, is_hyperapp, package_specific_features) = + if let Some(filename) = entry_path.file_name() { + if let Some(maybe_params) = + build_parameters.remove(&filename.to_string_lossy().to_string()) + { + ( + maybe_params.local_dependencies.unwrap_or_default(), + maybe_params.is_hyperapp.unwrap_or_default(), + maybe_params.features.unwrap_or_default(), + ) + } else { + (vec![], false, vec![]) + } } else { (vec![], false, vec![]) - } + }; + let package_specific_features = if package_specific_features.is_empty() { + features.clone() + } else if package_specific_features.contains(&"caller-utils".to_string()) { + // build without caller-utils flag, which will fail but will + // also create caller-utils crate (required for succeeding build) + let _ = build_and_zip_package( + entry_path.clone(), + child_pkg_path.to_str().unwrap(), + skip_frontend, + &features, + local_dependency_array.clone(), + is_hyperapp, + ); + format!("{features},{}", package_specific_features.join(",")) } else { - (vec![], false, vec![]) + format!("{features},{}", package_specific_features.join(",")) }; - let package_specific_features = if package_specific_features.is_empty() { - features.clone() - } else if package_specific_features.contains(&"caller-utils".to_string()) { - // build without caller-utils flag, which will fail but will - // also create caller-utils crate (required for succeeding build) - let _ = build_and_zip_package( - entry_path.clone(), - child_pkg_path.to_str().unwrap(), - skip_frontend, - &features, - local_dependency_array.clone(), + Some(( + entry_path, + child_pkg_path.to_string_lossy().to_string(), + package_specific_features, + local_dependency_array, is_hyperapp, - ); - format!("{features},{}", package_specific_features.join(",")) - } else { - format!("{features},{}", package_specific_features.join(",")) - }; - Some(( - entry_path, - child_pkg_path.to_string_lossy().to_string(), - package_specific_features, - local_dependency_array, - is_hyperapp, - )) - }) - .collect(); + )) + }) + .collect(); let results: Vec)>> = if parallel { package_build_info .into_par_iter() - .map(|(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { - build_and_zip_package( - entry_path, - &child_pkg_path, - skip_frontend, - &package_features, - local_deps, - is_hyperapp, - ) - }) + .map( + |(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { + build_and_zip_package( + entry_path, + &child_pkg_path, + skip_frontend, + &package_features, + local_deps, + is_hyperapp, + ) + }, + ) .collect() } else { package_build_info .into_iter() - .map(|(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { - build_and_zip_package( - entry_path, - &child_pkg_path, - skip_frontend, - &package_features, - local_deps, - is_hyperapp, - ) - }) + .map( + |(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { + build_and_zip_package( + entry_path, + &child_pkg_path, + skip_frontend, + &package_features, + local_deps, + is_hyperapp, + ) + }, + ) .collect() }; From 230f3d1bfe1446b0123060547de9fd6b029df4f0 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 17 Oct 2025 12:30:44 -0700 Subject: [PATCH 3/4] build-packages: always use rayon parallelism --- scripts/build-packages/src/main.rs | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/scripts/build-packages/src/main.rs b/scripts/build-packages/src/main.rs index 052c33916..a788f3ea1 100644 --- a/scripts/build-packages/src/main.rs +++ b/scripts/build-packages/src/main.rs @@ -114,12 +114,6 @@ fn main() -> anyhow::Result<()> { .help("Set output filename (default: packages-{features}.zip)") .action(clap::ArgAction::Set), ) - .arg( - Arg::new("PARALLEL") - .long("parallel") - .help("Build packages in parallel (faster but uses more resources)") - .action(clap::ArgAction::SetTrue), - ) .get_matches(); // hyperdrive/target/debug/build-package @@ -144,7 +138,6 @@ fn main() -> anyhow::Result<()> { let features = features.join(","); let skip_frontend = matches.get_flag("SKIP_FRONTEND"); - let parallel = matches.get_flag("PARALLEL"); let build_parameters = fs::read(hyperdrive_dir.join("packages-build-parameters.json"))?; let build_parameters: HashMap = @@ -221,7 +214,8 @@ fn main() -> anyhow::Result<()> { }) .collect(); - let results: Vec)>> = if parallel { + // Build in parallel + let results: Vec)>> = package_build_info .into_par_iter() .map( @@ -236,24 +230,7 @@ fn main() -> anyhow::Result<()> { ) }, ) - .collect() - } else { - package_build_info - .into_iter() - .map( - |(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { - build_and_zip_package( - entry_path, - &child_pkg_path, - skip_frontend, - &package_features, - local_deps, - is_hyperapp, - ) - }, - ) - .collect() - }; + .collect(); let mut file_to_metadata = std::collections::HashMap::new(); From 1440ff5513235959eb3dc4479660c77e5c910d84 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 19:31:06 +0000 Subject: [PATCH 4/4] Format Rust code using rustfmt --- scripts/build-packages/src/main.rs | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/scripts/build-packages/src/main.rs b/scripts/build-packages/src/main.rs index a788f3ea1..d55176f88 100644 --- a/scripts/build-packages/src/main.rs +++ b/scripts/build-packages/src/main.rs @@ -215,22 +215,21 @@ fn main() -> anyhow::Result<()> { .collect(); // Build in parallel - let results: Vec)>> = - package_build_info - .into_par_iter() - .map( - |(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { - build_and_zip_package( - entry_path, - &child_pkg_path, - skip_frontend, - &package_features, - local_deps, - is_hyperapp, - ) - }, - ) - .collect(); + let results: Vec)>> = package_build_info + .into_par_iter() + .map( + |(entry_path, child_pkg_path, package_features, local_deps, is_hyperapp)| { + build_and_zip_package( + entry_path, + &child_pkg_path, + skip_frontend, + &package_features, + local_deps, + is_hyperapp, + ) + }, + ) + .collect(); let mut file_to_metadata = std::collections::HashMap::new();