From 900787d44552b700bf0e40016d72e0520f9bac7e Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 11:40:25 -0700 Subject: [PATCH] feat(cli): #238 two-stage compile-many primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `fbuild compile-many --board --framework-jobs N --sketch-jobs M ...` which compiles many sketches against the same board with framework + library archives built **once** (stage 1) and per-sketch compile+link fanned out across a thread pool (stage 2). Routes through the existing `fbuild-build` orchestrator (`get_orchestrator(platform)`) so every platform automatically benefits without re-implementing platform logic. Independent parallelism knobs let memory-heavy stage 1 stay modest on constrained runners (`--framework-jobs` defaults to `min(cores, 2)`) while stage 2's per-worker memory footprint stays tiny enough to crank up (`--sketch-jobs` defaults to `cores`). Each stage-2 worker invokes the orchestrator with `jobs=1` since per-sketch work is single-TU and the outer thread pool already saturates cores. Concurrent-safety review: - Per-sketch output dirs are unique by construction (`/.fbuild/build///`); each sketch in the input list has its own `project_dir`, so no two workers race on `firmware.elf`. - The zccache compile-cache wrapper is lock-free on the hot path — fbuild adds no in-process locks around it (see `crates/fbuild-build/src/zccache.rs`), and concurrent stage-2 workers contend only against the zccache daemon's own SQLite-WAL concurrency model, well below the parallelism cap we set. Tests (`crates/fbuild-build/tests/compile_many_two_stage.rs`) inject a mock `SketchBuilder` and assert: - Stage 1 runs exactly once across N sketches; stage 2 produces N-1 results. - Results are returned in input order regardless of completion order. - Stage 2 workers run truly concurrently (Barrier-of-size-N proves real parallelism — would deadlock under serial execution). - Per-sketch firmware paths are unique (HashSet insert is asserted per build). - Stage-1 failure short-circuits stage 2. - Single sketch falls through stage 1 only. The numeric perf gates in the issue (warm <= 80 s, cold <= 110 s on ubuntu-latest) are validated by the benchmark workflow after this lands and a release ships; cannot be measured locally. The benchmark.yml workflow update is a separate PR on the orphan `bench/fastled-examples` branch and is explicitly out of scope here. Closes #238 partially (CLI primitive + tests; bench workflow follows). --- Cargo.lock | 1 + crates/fbuild-build/src/README.md | 1 + crates/fbuild-build/src/compile_many.rs | 642 ++++++++++++++++++ crates/fbuild-build/src/lib.rs | 1 + .../tests/compile_many_two_stage.rs | 344 ++++++++++ crates/fbuild-cli/Cargo.toml | 1 + crates/fbuild-cli/src/main.rs | 160 +++++ 7 files changed, 1150 insertions(+) create mode 100644 crates/fbuild-build/src/compile_many.rs create mode 100644 crates/fbuild-build/tests/compile_many_two_stage.rs diff --git a/Cargo.lock b/Cargo.lock index 0d7f93ab..92cbaf07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,6 +886,7 @@ dependencies = [ "blake3", "clap", "ctrlc", + "fbuild-build", "fbuild-config", "fbuild-core", "fbuild-deploy", diff --git a/crates/fbuild-build/src/README.md b/crates/fbuild-build/src/README.md index 09683fa6..a61488f7 100644 --- a/crates/fbuild-build/src/README.md +++ b/crates/fbuild-build/src/README.md @@ -13,6 +13,7 @@ Build orchestration for all supported embedded platforms. - **`compile_database.rs`** -- Generates `compile_commands.json` for clangd/IDE support - **`build_output.rs`** -- Uniform build log formatting across all platforms - **`zccache.rs`** -- Optional zccache compiler cache wrapper integration +- **`compile_many.rs`** -- Two-stage primitive for batched sketch builds (FastLED/fbuild#238): framework + libs built once with `--framework-jobs`, then per-sketch compile + link fanned out across `--sketch-jobs` workers ## Native `extra_scripts` Boundary diff --git a/crates/fbuild-build/src/compile_many.rs b/crates/fbuild-build/src/compile_many.rs new file mode 100644 index 00000000..aeb7b9a0 --- /dev/null +++ b/crates/fbuild-build/src/compile_many.rs @@ -0,0 +1,642 @@ +//! Two-stage `compile-many` primitive (FastLED/fbuild#238). +//! +//! Compiles a list of sketches against the same board with the framework + +//! library archives built **once**, then fans out per-sketch compile + link +//! across a thread pool. +//! +//! ## Design +//! +//! The naive "loop over `fbuild build`" strategy pays the orchestrator +//! startup cost N times and serializes work that could overlap. The +//! `compile-many` primitive splits that work into two stages with +//! independent parallelism knobs: +//! +//! - **Stage 1** runs the orchestrator once for the first sketch with +//! `--framework-jobs` workers driving intra-build parallelism (e.g. +//! parallel compile of framework `.cpp` files). This is the +//! memory-heavy stage; keep the knob modest on constrained runners. +//! +//! - **Stage 2** fans out the remaining sketches across `--sketch-jobs` +//! workers. Each worker reuses the framework / library archives written +//! to disk by stage 1 (via the orchestrator's warm-build fast path and +//! zccache), and only pays for `sketch.cpp -> .o -> link` per sketch. +//! Each worker calls the orchestrator with `jobs = 1` since per-sketch +//! work is small and the outer thread pool already saturates cores. +//! +//! ## Concurrent-safety +//! +//! Stage-2 workers operate concurrently on distinct project directories. +//! Two guarantees keep that race-free: +//! +//! 1. **Per-sketch output directories are unique.** The orchestrator +//! derives the build root from `/.fbuild/build///`, +//! and each sketch in the input list has its own `project_dir`, so +//! no two workers can touch the same `firmware.elf`. +//! +//! 2. **The zccache compile-cache wrapper is lock-free on the hot path.** +//! Each worker invokes `zccache wrap ...` as a child process; +//! upstream zccache uses SQLite WAL + content-addressed object files +//! so concurrent reads of the same key never block, and concurrent +//! writes of distinct keys never block. fbuild itself adds no +//! in-process locks around zccache (see `crates/fbuild-build/src/zccache.rs`), +//! so stage-2 contention is bounded by the zccache daemon's own +//! concurrency model — well below the parallelism cap we set. +//! +//! ## Routing through the existing orchestrator +//! +//! Each sketch is built through `fbuild_build::get_orchestrator(platform)`, +//! so all platform-specific behavior (toolchain resolution, framework +//! installation, LDF, link flags, size reporting) lives in one place and +//! `compile-many` automatically picks up future per-platform work without +//! re-implementing it. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; +use std::time::Instant; + +use fbuild_core::{BuildProfile, FbuildError, Platform, Result}; + +use crate::{get_orchestrator, BuildParams, BuildResult}; + +/// Default for `--framework-jobs` when not specified: `min(cores, 2)`. +/// +/// Framework compilation is memory-heavy; a 2-core `ubuntu-latest` runner +/// can OOM with more. Beefier runners benefit from manually cranking this +/// up via the CLI flag. +pub fn default_framework_jobs() -> usize { + let cores = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + cores.clamp(1, 2) +} + +/// Default for `--sketch-jobs` when not specified: `cores`. +/// +/// Per-sketch work (sketch.cpp compile + link against pre-built archives) +/// has a tiny memory footprint per worker, so fanning out to one worker per +/// physical core is safe even on small runners. +pub fn default_sketch_jobs() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .max(1) +} + +/// Request parameters for [`compile_many`]. +#[derive(Debug, Clone)] +pub struct CompileManyRequest { + /// Board id (e.g. "uno", "teensy41"). Used to pick the matching + /// environment within each sketch's `platformio.ini` and to dispatch + /// to the right platform orchestrator. + pub board: String, + /// One project directory per sketch. Each must contain a + /// `platformio.ini` with an environment whose `board = ` (or + /// an environment literally named ``). + pub sketches: Vec, + /// Parallelism for stage 1 (framework + library compile). + /// `None` -> [`default_framework_jobs`]. + pub framework_jobs: Option, + /// Parallelism for stage 2 (per-sketch compile + link). + /// `None` -> [`default_sketch_jobs`]. + pub sketch_jobs: Option, + /// Build profile (release / quick). + pub profile: BuildProfile, + /// Verbose compiler output. + pub verbose: bool, +} + +/// Result for a single sketch. +#[derive(Debug, Clone)] +pub struct SketchResult { + /// Sketch project directory (matches the request entry). + pub sketch: PathBuf, + /// Resolved environment name within the sketch's `platformio.ini`. + pub env_name: String, + /// Whether the sketch built successfully. + pub success: bool, + /// Path to the produced firmware (hex/bin/uf2) when successful. + pub firmware_path: Option, + /// Path to the produced ELF when available. + pub elf_path: Option, + /// Build wall-clock seconds. + pub build_time_secs: f64, + /// Path to the per-sketch log file (text dump of `BuildLog`). + pub log_path: Option, + /// Human-readable summary message. + pub message: String, + /// Stage that produced this result. + pub stage: Stage, +} + +/// Which stage compiled a sketch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Stage { + /// First sketch: built sequentially to warm framework/library archives. + Stage1Framework, + /// Subsequent sketches: built concurrently against pre-built archives. + Stage2Sketch, +} + +/// Final result of a `compile-many` invocation. +#[derive(Debug, Clone)] +pub struct CompileManyResult { + /// One entry per requested sketch, in input order. + pub results: Vec, + /// `true` iff every sketch built successfully. + pub all_success: bool, + /// Number of sketches processed by stage 1. + pub stage1_count: usize, + /// Number of sketches processed by stage 2. + pub stage2_count: usize, + /// Wall-clock for stage 1. + pub stage1_secs: f64, + /// Wall-clock for stage 2. + pub stage2_secs: f64, + /// Wall-clock for the entire `compile-many` call. + pub total_secs: f64, +} + +impl CompileManyResult { + /// Lookup result by sketch path. + pub fn get(&self, sketch: &Path) -> Option<&SketchResult> { + self.results.iter().find(|r| r.sketch == sketch) + } + + /// Map of sketch path -> success/log_path, suitable for bench summaries. + pub fn as_map(&self) -> HashMap)> { + self.results + .iter() + .map(|r| (r.sketch.clone(), (r.success, r.log_path.clone()))) + .collect() + } +} + +/// Resolve the environment name inside `platformio.ini` for the given board. +/// +/// Priority: +/// 1. An environment literally named ``. +/// 2. The first environment whose `board = `. +/// +/// Returns `Err` when neither is found — this is a contract violation that +/// should surface immediately rather than guessing. +fn resolve_env_for_board(project_dir: &Path, board: &str) -> Result { + let ini_path = project_dir.join("platformio.ini"); + let config = fbuild_config::PlatformIOConfig::from_path(&ini_path)?; + + if config.has_environment(board) { + return Ok(board.to_string()); + } + for env in config.get_environments() { + if let Ok(env_config) = config.get_env_config(env) { + if env_config.get("board").map(|s| s.as_str()) == Some(board) { + return Ok(env.to_string()); + } + } + } + Err(FbuildError::ConfigError(format!( + "no environment in {} matches board '{}' (looked for [env:{}] or board={})", + ini_path.display(), + board, + board, + board + ))) +} + +/// Determine the platform for a board id. +fn platform_for_board(board: &str) -> Result { + let cfg = fbuild_config::BoardConfig::from_board_id(board, &HashMap::new())?; + cfg.platform().ok_or_else(|| { + FbuildError::ConfigError(format!( + "could not determine platform for board '{}' (mcu '{}')", + board, cfg.mcu + )) + }) +} + +/// Build a single sketch through the platform orchestrator. +/// +/// `jobs` controls intra-build parallelism (passed through to the +/// orchestrator's per-build thread pool). +fn build_one_sketch( + sketch: &Path, + env_name: &str, + platform: Platform, + profile: BuildProfile, + jobs: usize, + verbose: bool, + stage: Stage, +) -> SketchResult { + let start = Instant::now(); + let build_dir = fbuild_packages::Cache::new(sketch).get_build_dir(env_name, profile); + let params = BuildParams { + project_dir: sketch.to_path_buf(), + env_name: env_name.to_string(), + clean: false, + profile, + build_dir, + verbose, + jobs: Some(jobs), + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: true, + src_dir: None, + pio_env: Default::default(), + extra_build_flags: Vec::new(), + watch_set_cache: None, + }; + + let outcome = match get_orchestrator(platform) { + Ok(orch) => orch.build(¶ms), + Err(e) => Err(e), + }; + + match outcome { + Ok(br) => { + let BuildResult { + success, + firmware_path, + elf_path, + build_time_secs, + message, + build_log, + .. + } = br; + let log_path = write_log_file(sketch, env_name, profile, build_log); + SketchResult { + sketch: sketch.to_path_buf(), + env_name: env_name.to_string(), + success, + firmware_path, + elf_path, + build_time_secs, + log_path, + message, + stage, + } + } + Err(e) => SketchResult { + sketch: sketch.to_path_buf(), + env_name: env_name.to_string(), + success: false, + firmware_path: None, + elf_path: None, + build_time_secs: start.elapsed().as_secs_f64(), + log_path: None, + message: format!("build error: {}", e), + stage, + }, + } +} + +/// Persist a per-sketch log file alongside the build artifacts so the +/// bench summary / caller can fish out the full output without holding it +/// in memory across N parallel workers. +fn write_log_file( + sketch: &Path, + env_name: &str, + profile: BuildProfile, + build_log: fbuild_core::BuildLog, +) -> Option { + let build_dir = fbuild_packages::Cache::new(sketch).get_build_dir(env_name, profile); + if std::fs::create_dir_all(&build_dir).is_err() { + return None; + } + let log_path = build_dir.join("compile_many.log"); + let body = build_log.into_lines().join("\n"); + match std::fs::write(&log_path, body) { + Ok(()) => Some(log_path), + Err(_) => None, + } +} + +/// Inputs to a single sketch build (consumed by [`SketchBuilder`]). +#[derive(Debug, Clone)] +pub struct SketchBuildInputs { + pub sketch: PathBuf, + pub env_name: String, + pub platform: Platform, + pub profile: BuildProfile, + pub jobs: usize, + pub verbose: bool, + pub stage: Stage, +} + +/// Trait used by [`compile_many_with`] to run a single sketch. Tests +/// inject a mock implementation that records stage / concurrency / output +/// path uniqueness without dragging in a real toolchain. +pub trait SketchBuilder: Sync { + fn build(&self, inputs: SketchBuildInputs) -> SketchResult; +} + +/// Production [`SketchBuilder`] that drives the real platform +/// orchestrator. This is the only place `get_orchestrator` is touched +/// from `compile_many`, so tests can swap it out wholesale. +pub struct OrchestratorBuilder; + +impl SketchBuilder for OrchestratorBuilder { + fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + build_one_sketch( + &inputs.sketch, + &inputs.env_name, + inputs.platform, + inputs.profile, + inputs.jobs, + inputs.verbose, + inputs.stage, + ) + } +} + +/// Run the two-stage `compile-many` flow. +/// +/// Returns once every sketch has been attempted. Individual sketch failures +/// do not short-circuit subsequent sketches — the caller inspects +/// [`CompileManyResult::all_success`] / [`CompileManyResult::results`]. +pub fn compile_many(req: CompileManyRequest) -> Result { + compile_many_with(req, &OrchestratorBuilder) +} + +/// Like [`compile_many`] but parameterized over the [`SketchBuilder`] +/// used to actually build each sketch. Public for tests; production +/// callers should use [`compile_many`]. +pub fn compile_many_with( + req: CompileManyRequest, + builder: &dyn SketchBuilder, +) -> Result { + if req.sketches.is_empty() { + return Err(FbuildError::Other( + "compile_many: at least one sketch is required".to_string(), + )); + } + + let framework_jobs = req + .framework_jobs + .unwrap_or_else(default_framework_jobs) + .max(1); + let sketch_jobs = req.sketch_jobs.unwrap_or_else(default_sketch_jobs).max(1); + let platform = platform_for_board(&req.board)?; + + // Pre-resolve env names + assert each sketch dir exists. Doing this + // up front means we never half-build the batch and leave one worker + // crashing later with a stale path. + let mut resolved: Vec<(PathBuf, String)> = Vec::with_capacity(req.sketches.len()); + for sketch in &req.sketches { + if !sketch.is_dir() { + return Err(FbuildError::Other(format!( + "sketch project_dir does not exist: {}", + sketch.display() + ))); + } + let env = resolve_env_for_board(sketch, &req.board)?; + resolved.push((sketch.clone(), env)); + } + + let total_start = Instant::now(); + + // -------- Stage 1: build the first sketch sequentially. -------- + // + // This warms the framework / library archives and the orchestrator's + // warm-build fingerprint cache on disk. Subsequent stage-2 workers + // hit those caches instead of re-building the framework. + let stage1_start = Instant::now(); + let (first_sketch, first_env) = resolved[0].clone(); + let first_result = builder.build(SketchBuildInputs { + sketch: first_sketch, + env_name: first_env, + platform, + profile: req.profile, + jobs: framework_jobs, + verbose: req.verbose, + stage: Stage::Stage1Framework, + }); + let stage1_secs = stage1_start.elapsed().as_secs_f64(); + + // If stage 1 failed there is no point fanning out — every stage-2 + // worker would re-run the framework build (which we just proved + // broken) and report the same error. Return what we have so far. + if !first_result.success { + let total_secs = total_start.elapsed().as_secs_f64(); + return Ok(CompileManyResult { + results: vec![first_result], + all_success: false, + stage1_count: 1, + stage2_count: 0, + stage1_secs, + stage2_secs: 0.0, + total_secs, + }); + } + + // -------- Stage 2: fan out the remaining sketches in parallel. -------- + let stage2_start = Instant::now(); + let rest: Vec<(PathBuf, String)> = resolved[1..].to_vec(); + let stage2_results = if rest.is_empty() { + Vec::new() + } else { + run_stage2( + &rest, + platform, + req.profile, + sketch_jobs, + req.verbose, + builder, + ) + }; + let stage2_secs = stage2_start.elapsed().as_secs_f64(); + + let mut results = Vec::with_capacity(req.sketches.len()); + results.push(first_result); + results.extend(stage2_results); + + let all_success = results.iter().all(|r| r.success); + let stage2_count = results.len().saturating_sub(1); + let total_secs = total_start.elapsed().as_secs_f64(); + + Ok(CompileManyResult { + results, + all_success, + stage1_count: 1, + stage2_count, + stage1_secs, + stage2_secs, + total_secs, + }) +} + +/// Run stage-2 workers across `rest` with up to `sketch_jobs` concurrent +/// threads. Preserves input order in the returned `Vec`. +fn run_stage2( + rest: &[(PathBuf, String)], + platform: Platform, + profile: BuildProfile, + sketch_jobs: usize, + verbose: bool, + builder: &dyn SketchBuilder, +) -> Vec { + let total = rest.len(); + let cap = sketch_jobs.min(total).max(1); + tracing::info!( + "compile-many stage 2: {} sketches across {} workers", + total, + cap + ); + + // Work queue: indices into `rest`. We dispatch by index so the result + // slot lives at a stable position regardless of completion order. + let next = AtomicUsize::new(0); + let mut results: Vec> = (0..total).map(|_| None).collect(); + let results_slot: Vec>> = + results.iter_mut().map(|_| Mutex::new(None)).collect(); + + std::thread::scope(|scope| { + let handles: Vec<_> = (0..cap) + .map(|_| { + let next = &next; + let results_slot = &results_slot; + scope.spawn(move || loop { + let idx = next.fetch_add(1, Ordering::Relaxed); + if idx >= rest.len() { + break; + } + let entry = &rest[idx]; + let (sketch, env_name) = (&entry.0, &entry.1); + let res = builder.build(SketchBuildInputs { + sketch: sketch.clone(), + env_name: env_name.clone(), + platform, + profile, + // Per-sketch work is single-TU; framework archives + // are already pre-built, so jobs=1 keeps memory + // per worker minimal. + jobs: 1, + verbose, + stage: Stage::Stage2Sketch, + }); + *results_slot[idx].lock().unwrap() = Some(res); + }) + }) + .collect(); + for h in handles { + let _ = h.join(); + } + }); + + results_slot + .into_iter() + .map(|slot| slot.into_inner().unwrap().expect("worker filled slot")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_framework_jobs_is_at_least_one_and_at_most_two() { + let n = default_framework_jobs(); + assert!(n >= 1, "framework jobs must be >= 1"); + assert!( + n <= 2, + "framework jobs default must be capped at 2 (got {n})" + ); + } + + #[test] + fn default_sketch_jobs_is_at_least_one() { + assert!(default_sketch_jobs() >= 1); + } + + #[test] + fn resolve_env_picks_literal_env_name() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + assert_eq!(resolve_env_for_board(tmp.path(), "uno").unwrap(), "uno"); + } + + #[test] + fn resolve_env_falls_back_to_board_match() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:my_custom]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + assert_eq!( + resolve_env_for_board(tmp.path(), "uno").unwrap(), + "my_custom" + ); + } + + #[test] + fn resolve_env_errors_on_no_match() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + assert!(resolve_env_for_board(tmp.path(), "teensy41").is_err()); + } + + #[test] + fn platform_for_board_uno_is_avr() { + let p = platform_for_board("uno").unwrap(); + assert_eq!(p, Platform::AtmelAvr); + } + + #[test] + fn empty_sketch_list_errors_out() { + let req = CompileManyRequest { + board: "uno".to_string(), + sketches: Vec::new(), + framework_jobs: None, + sketch_jobs: None, + profile: BuildProfile::Release, + verbose: false, + }; + assert!(compile_many(req).is_err()); + } + + #[test] + fn missing_sketch_dir_errors_out() { + let tmp = tempfile::tempdir().unwrap(); + let missing = tmp.path().join("nope"); + let req = CompileManyRequest { + board: "uno".to_string(), + sketches: vec![missing], + framework_jobs: None, + sketch_jobs: None, + profile: BuildProfile::Release, + verbose: false, + }; + assert!(compile_many(req).is_err()); + } + + #[test] + fn sketch_without_matching_board_errors_out() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + let req = CompileManyRequest { + board: "esp32dev".to_string(), + sketches: vec![tmp.path().to_path_buf()], + framework_jobs: None, + sketch_jobs: None, + profile: BuildProfile::Release, + verbose: false, + }; + assert!(compile_many(req).is_err()); + } +} diff --git a/crates/fbuild-build/src/lib.rs b/crates/fbuild-build/src/lib.rs index 7aaaa1f1..c7da3332 100644 --- a/crates/fbuild-build/src/lib.rs +++ b/crates/fbuild-build/src/lib.rs @@ -10,6 +10,7 @@ pub mod build_fingerprint; pub mod build_output; pub mod ch32v; pub mod compile_database; +pub mod compile_many; pub mod compiler; pub mod esp32; pub mod esp8266; diff --git a/crates/fbuild-build/tests/compile_many_two_stage.rs b/crates/fbuild-build/tests/compile_many_two_stage.rs new file mode 100644 index 00000000..6210861e --- /dev/null +++ b/crates/fbuild-build/tests/compile_many_two_stage.rs @@ -0,0 +1,344 @@ +//! Integration tests for the two-stage `compile-many` flow +//! (FastLED/fbuild#238). +//! +//! These tests inject a mock [`SketchBuilder`] so we exercise the +//! orchestration layer (stage counts, parallelism, output-path +//! uniqueness, input ordering) without dragging in a real toolchain. +//! Real-toolchain coverage lives in `avr_build.rs` etc. and is +//! `#[ignore]`-gated to keep `uv run test` fast. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier, Mutex}; +use std::time::Duration; + +use fbuild_build::compile_many::{ + compile_many_with, CompileManyRequest, SketchBuildInputs, SketchBuilder, SketchResult, Stage, +}; +use fbuild_core::BuildProfile; + +/// Test sketch root with a minimal `platformio.ini`. Used as a +/// `project_dir` parameter — the mock builder does not read it, but +/// `compile_many` validates the file's `[env:...]` matches the requested +/// board. +fn make_sketch(parent: &Path, name: &str, board: &str) -> PathBuf { + let dir = parent.join(name); + std::fs::create_dir_all(&dir).expect("mkdir sketch"); + let platform = match board { + "uno" => "atmelavr", + "teensy41" => "teensy", + _ => "atmelavr", + }; + std::fs::write( + dir.join("platformio.ini"), + format!("[env:{board}]\nplatform = {platform}\nboard = {board}\nframework = arduino\n"), + ) + .expect("write platformio.ini"); + dir +} + +/// Mock builder that: +/// - Records every (sketch, stage, jobs) tuple it is asked to build. +/// - Writes a unique sentinel file under the canonical per-sketch +/// build_dir so we can assert no two workers race on `firmware.elf`. +/// - Optionally synchronizes stage-2 workers on a `Barrier` to prove +/// real concurrency (not just queued serial execution). +struct MockBuilder { + /// (sketch_path, env_name, stage, jobs). + calls: Mutex>, + /// Sentinel firmware paths created during the test, used to assert + /// per-sketch output paths are unique. + firmware_paths: Mutex>, + /// Optional barrier; if set, every stage-2 invocation waits on it + /// before "completing". A successful `wait()` proves N workers + /// were running concurrently. + stage2_barrier: Option>, + /// Bumped on every stage-2 wait completion; lets the test verify the + /// barrier did, in fact, fire. + stage2_wait_count: AtomicUsize, +} + +impl MockBuilder { + fn new() -> Self { + Self { + calls: Mutex::new(Vec::new()), + firmware_paths: Mutex::new(HashSet::new()), + stage2_barrier: None, + stage2_wait_count: AtomicUsize::new(0), + } + } + + fn with_barrier(barrier: Arc) -> Self { + Self { + calls: Mutex::new(Vec::new()), + firmware_paths: Mutex::new(HashSet::new()), + stage2_barrier: Some(barrier), + stage2_wait_count: AtomicUsize::new(0), + } + } + + fn calls(&self) -> Vec<(PathBuf, String, Stage, usize)> { + self.calls.lock().unwrap().clone() + } + + fn firmware_paths(&self) -> HashSet { + self.firmware_paths.lock().unwrap().clone() + } +} + +impl SketchBuilder for MockBuilder { + fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + // Synthesize the canonical per-sketch firmware path the same + // way the real orchestrator does. We deliberately use the + // same naming convention so the uniqueness assertion is + // meaningful — two workers racing on identical paths is the + // bug we are trying to rule out. + let build_dir = inputs + .sketch + .join(".fbuild") + .join("build") + .join(&inputs.env_name) + .join(match inputs.profile { + BuildProfile::Release => "release", + BuildProfile::Quick => "quick", + }); + std::fs::create_dir_all(&build_dir).expect("mkdir build_dir"); + let firmware = build_dir.join("firmware.elf"); + // Write a tiny sentinel and assert no collision happened. + std::fs::write(&firmware, inputs.sketch.to_string_lossy().as_bytes()) + .expect("write firmware sentinel"); + { + let mut paths = self.firmware_paths.lock().unwrap(); + assert!( + paths.insert(firmware.clone()), + "duplicate firmware path observed (race on {})", + firmware.display() + ); + } + + { + let mut calls = self.calls.lock().unwrap(); + calls.push(( + inputs.sketch.clone(), + inputs.env_name.clone(), + inputs.stage, + inputs.jobs, + )); + } + + // Force real overlap among stage-2 workers when a barrier is + // wired up: every worker blocks here until N peers arrive. + if inputs.stage == Stage::Stage2Sketch { + if let Some(ref b) = self.stage2_barrier { + b.wait(); + self.stage2_wait_count.fetch_add(1, Ordering::Relaxed); + } + } + + SketchResult { + sketch: inputs.sketch.clone(), + env_name: inputs.env_name.clone(), + success: true, + firmware_path: Some(firmware), + elf_path: None, + build_time_secs: 0.0, + log_path: None, + message: "mock build ok".to_string(), + stage: inputs.stage, + } + } +} + +fn make_request( + sketches: Vec, + framework_jobs: usize, + sketch_jobs: usize, +) -> CompileManyRequest { + CompileManyRequest { + board: "uno".to_string(), + sketches, + framework_jobs: Some(framework_jobs), + sketch_jobs: Some(sketch_jobs), + profile: BuildProfile::Release, + verbose: false, + } +} + +/// AC: stage 1 runs exactly once across N sketches, stage 2 produces +/// N-1 firmware artifacts, and per-sketch output paths are unique. +#[test] +fn stage1_runs_exactly_once_and_stage2_handles_the_rest() { + let tmp = tempfile::tempdir().unwrap(); + let sketches: Vec = (0..5) + .map(|i| make_sketch(tmp.path(), &format!("sketch{i}"), "uno")) + .collect(); + + let mock = MockBuilder::new(); + let result = + compile_many_with(make_request(sketches.clone(), 1, 4), &mock).expect("compile_many"); + + assert!(result.all_success, "all mock builds should succeed"); + assert_eq!(result.results.len(), 5); + assert_eq!(result.stage1_count, 1, "exactly one stage-1 invocation"); + assert_eq!(result.stage2_count, 4, "four stage-2 invocations"); + + let calls = mock.calls(); + let stage1: Vec<_> = calls + .iter() + .filter(|(_, _, s, _)| *s == Stage::Stage1Framework) + .collect(); + let stage2: Vec<_> = calls + .iter() + .filter(|(_, _, s, _)| *s == Stage::Stage2Sketch) + .collect(); + assert_eq!(stage1.len(), 1, "framework stage called once"); + assert_eq!( + stage2.len(), + 4, + "sketch stage called once per remaining sketch" + ); + + // Stage-1 honors framework_jobs; stage-2 always passes jobs=1. + assert_eq!(stage1[0].3, 1, "framework_jobs=1 forwarded to stage 1"); + for (_, _, _, jobs) in &stage2 { + assert_eq!( + *jobs, 1, + "stage 2 workers always invoke orchestrator with jobs=1" + ); + } + + // Per-sketch output paths must be unique — the firmware_paths set + // is built by the mock builder, which `assert!`s uniqueness on + // insert. We also cross-check the count here as a safety net. + let firmwares = mock.firmware_paths(); + assert_eq!(firmwares.len(), sketches.len(), "one firmware per sketch"); +} + +/// AC: stage-1 results are placed at index 0; stage-2 results follow +/// input order regardless of completion order. +#[test] +fn results_are_returned_in_input_order() { + let tmp = tempfile::tempdir().unwrap(); + let sketches: Vec = (0..6) + .map(|i| make_sketch(tmp.path(), &format!("ordered{i}"), "uno")) + .collect(); + + let mock = MockBuilder::new(); + let result = compile_many_with(make_request(sketches.clone(), 1, 3), &mock).expect("ok"); + + for (i, r) in result.results.iter().enumerate() { + assert_eq!(r.sketch, sketches[i], "result {i} should match input order"); + } + assert_eq!(result.results[0].stage, Stage::Stage1Framework); + for r in &result.results[1..] { + assert_eq!(r.stage, Stage::Stage2Sketch); + } +} + +/// AC: with `sketch_jobs >= N`, all stage-2 workers run truly +/// concurrently — proven via a Barrier that would deadlock under serial +/// execution. The barrier counts each stage-2 worker as it crosses; a +/// successful test means N workers ran in parallel. +#[test] +fn stage2_workers_run_concurrently() { + let tmp = tempfile::tempdir().unwrap(); + let n_stage2 = 4; + let total = n_stage2 + 1; + let sketches: Vec = (0..total) + .map(|i| make_sketch(tmp.path(), &format!("concurrent{i}"), "uno")) + .collect(); + + // Barrier sized to exactly the stage-2 worker count. If + // `compile_many` ran them serially, `wait()` would block forever + // (only one worker at a time would arrive), and the test would + // hang. We add a wall-clock timeout below so a regression is + // a fast failure rather than a CI hang. + let barrier = Arc::new(Barrier::new(n_stage2)); + let mock = Arc::new(MockBuilder::with_barrier(Arc::clone(&barrier))); + let req = make_request(sketches, 1, n_stage2); + + let mock_for_thread = Arc::clone(&mock); + let handle = std::thread::spawn(move || compile_many_with(req, mock_for_thread.as_ref())); + + // Generous deadline; on a healthy machine the test completes in + // milliseconds. The deadline only fires if the barrier deadlocks + // because workers ran serially. + let start = std::time::Instant::now(); + let deadline = Duration::from_secs(10); + loop { + if handle.is_finished() { + break; + } + if start.elapsed() > deadline { + panic!( + "stage-2 deadlock: barrier expected {} concurrent workers but only {} arrived", + n_stage2, + mock.stage2_wait_count.load(Ordering::Relaxed) + ); + } + std::thread::sleep(Duration::from_millis(20)); + } + + let result = handle.join().expect("worker thread").expect("compile_many"); + assert!(result.all_success); + assert_eq!(result.stage2_count, n_stage2); + assert_eq!( + mock.stage2_wait_count.load(Ordering::Relaxed), + n_stage2, + "every stage-2 worker should have crossed the barrier" + ); +} + +/// AC: stage-1 failure short-circuits stage 2 — no point fanning out +/// when the framework build is broken, every stage-2 worker would just +/// repeat the same error. +#[test] +fn stage1_failure_skips_stage2() { + struct FailingStage1; + impl SketchBuilder for FailingStage1 { + fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + SketchResult { + sketch: inputs.sketch.clone(), + env_name: inputs.env_name, + success: inputs.stage != Stage::Stage1Framework, + firmware_path: None, + elf_path: None, + build_time_secs: 0.0, + log_path: None, + message: "mock failure".to_string(), + stage: inputs.stage, + } + } + } + + let tmp = tempfile::tempdir().unwrap(); + let sketches: Vec = (0..3) + .map(|i| make_sketch(tmp.path(), &format!("fail{i}"), "uno")) + .collect(); + let result = compile_many_with(make_request(sketches, 1, 2), &FailingStage1).expect("ok"); + assert!(!result.all_success); + assert_eq!(result.stage1_count, 1); + assert_eq!( + result.stage2_count, 0, + "stage 2 must be skipped on stage-1 failure" + ); + assert_eq!( + result.results.len(), + 1, + "only the failing stage-1 sketch is returned" + ); +} + +/// AC: a single sketch falls through stage 1 only — stage 2 is empty. +#[test] +fn single_sketch_runs_only_stage1() { + let tmp = tempfile::tempdir().unwrap(); + let sketch = make_sketch(tmp.path(), "only", "uno"); + let mock = MockBuilder::new(); + let result = compile_many_with(make_request(vec![sketch], 2, 4), &mock).expect("ok"); + assert!(result.all_success); + assert_eq!(result.stage1_count, 1); + assert_eq!(result.stage2_count, 0); + assert_eq!(result.results[0].stage, Stage::Stage1Framework); +} diff --git a/crates/fbuild-cli/Cargo.toml b/crates/fbuild-cli/Cargo.toml index c28f2ef6..d6264ce9 100644 --- a/crates/fbuild-cli/Cargo.toml +++ b/crates/fbuild-cli/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] fbuild-core = { path = "../fbuild-core" } +fbuild-build = { path = "../fbuild-build" } fbuild-config = { path = "../fbuild-config" } fbuild-deploy = { path = "../fbuild-deploy" } fbuild-library-select = { path = "../fbuild-library-select" } diff --git a/crates/fbuild-cli/src/main.rs b/crates/fbuild-cli/src/main.rs index d6dfa3de..2e499c8d 100644 --- a/crates/fbuild-cli/src/main.rs +++ b/crates/fbuild-cli/src/main.rs @@ -301,6 +301,38 @@ enum Commands { #[arg(long, conflicts_with = "explain")] json: bool, }, + /// Two-stage compile of many sketches against the same board + /// (FastLED/fbuild#238). Builds the framework + library archives once + /// (stage 1) and fans out per-sketch compile + link in parallel + /// (stage 2). Independent parallelism knobs for each stage so memory- + /// heavy framework work stays modest while sketch work saturates cores. + CompileMany { + /// Board id (e.g. "uno", "teensy41"). Used to dispatch to the + /// right platform orchestrator and to pick the matching + /// `[env:]` (or first env with `board = `) inside + /// each sketch's `platformio.ini`. + #[arg(long)] + board: String, + /// Parallelism for stage 1 (framework + library compile). When + /// omitted, defaults to `min(cores, 2)`. + #[arg(long)] + framework_jobs: Option, + /// Parallelism for stage 2 (per-sketch compile + link). When + /// omitted, defaults to `cores`. + #[arg(long)] + sketch_jobs: Option, + /// Build profile. + #[arg(long, group = "compile_many_profile")] + quick: bool, + #[arg(long, group = "compile_many_profile")] + release: bool, + /// Verbose compiler output. + #[arg(short, long)] + verbose: bool, + /// Sketch project directories (each must contain `platformio.ini`). + #[arg(required = true)] + sketches: Vec, + }, } /// Subcommands for `fbuild lnk`. @@ -441,6 +473,7 @@ const KNOWN_SUBCOMMANDS: &[&str] = &[ "clang-query", "test-emu", "lib-select", + "compile-many", ]; /// Rewrite `fbuild ...` → `fbuild ...` @@ -756,6 +789,26 @@ async fn main() { ); std::process::exit(exit); } + Some(Commands::CompileMany { + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + }) => { + run_compile_many( + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + ) + .await + } None => { // Default action: deploy with monitor (like Python fbuild) let project_dir = cli.project_dir.unwrap_or_else(|| ".".to_string()); @@ -1173,6 +1226,113 @@ async fn run_build( Ok(()) } +/// Handler for `fbuild compile-many` (FastLED/fbuild#238). +/// +/// Runs the two-stage compile-many primitive in-process via the existing +/// `fbuild-build` orchestrator. We bypass the daemon here on purpose: the +/// design goal of #238 is "one process, one toolchain load, one LDF run, +/// one framework build" — so all stage-1 + stage-2 work happens in this +/// process, parallelism is driven by the `compile_many` thread pool, and +/// no per-sketch daemon round-trip is incurred. +async fn run_compile_many( + board: String, + framework_jobs: Option, + sketch_jobs: Option, + quick: bool, + release: bool, + verbose: bool, + sketches: Vec, +) -> fbuild_core::Result<()> { + use fbuild_build::compile_many::{compile_many, CompileManyRequest, Stage}; + + let profile = if release { + fbuild_core::BuildProfile::Release + } else if quick { + fbuild_core::BuildProfile::Quick + } else { + // Default to release: matches `fbuild build`'s default profile so + // CI builds aren't silently dropped into quick mode. + fbuild_core::BuildProfile::Release + }; + + let sketches: Vec = + sketches.into_iter().map(std::path::PathBuf::from).collect(); + + let req = CompileManyRequest { + board: board.clone(), + sketches: sketches.clone(), + framework_jobs, + sketch_jobs, + profile, + verbose, + }; + + let effective_framework = req + .framework_jobs + .unwrap_or_else(fbuild_build::compile_many::default_framework_jobs); + let effective_sketch = req + .sketch_jobs + .unwrap_or_else(fbuild_build::compile_many::default_sketch_jobs); + println!( + "compile-many: board={} sketches={} framework_jobs={} sketch_jobs={}", + board, + sketches.len(), + effective_framework, + effective_sketch, + ); + + // `compile_many` is fully synchronous (CPU-bound). Run it on a + // blocking pool so we don't tie up the tokio runtime thread. + let result = tokio::task::spawn_blocking(move || compile_many(req)) + .await + .map_err(|e| { + fbuild_core::FbuildError::Other(format!("compile-many task panicked: {e}")) + })??; + + // Per-sketch result map suitable for the bench summary. + println!(); + println!("compile-many results:"); + for r in &result.results { + let stage_label = match r.stage { + Stage::Stage1Framework => "stage1", + Stage::Stage2Sketch => "stage2", + }; + let status = if r.success { "OK" } else { "FAIL" }; + let log_str = r + .log_path + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_else(|| "-".to_string()); + println!( + " [{}] {} ({:.2}s) {} log={} {}", + stage_label, + status, + r.build_time_secs, + r.sketch.display(), + log_str, + r.message, + ); + } + println!(); + println!( + "compile-many summary: stage1={}/{:.2}s stage2={}/{:.2}s total={:.2}s", + result.stage1_count, + result.stage1_secs, + result.stage2_count, + result.stage2_secs, + result.total_secs, + ); + + if !result.all_success { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "compile-many: {}/{} sketches failed", + result.results.iter().filter(|r| !r.success).count(), + result.results.len(), + ))); + } + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum CliEmulatorKind { Qemu,