From ed9a8334461327f8933c28d2a5a321df2db8311f Mon Sep 17 00:00:00 2001 From: zackees Date: Mon, 13 Apr 2026 15:50:29 -0700 Subject: [PATCH 1/3] Add extra_scripts flag runtime bridge --- crates/fbuild-build/Cargo.toml | 2 +- crates/fbuild-build/src/avr/avr_linker.rs | 5 +- crates/fbuild-build/src/ch32v/ch32v_linker.rs | 5 +- crates/fbuild-build/src/compile_database.rs | 41 +- crates/fbuild-build/src/esp32/esp32_linker.rs | 5 +- crates/fbuild-build/src/esp32/orchestrator.rs | 71 +++- .../src/esp8266/esp8266_linker.rs | 5 +- crates/fbuild-build/src/flag_overlay.rs | 218 +++++++++++ .../src/generic_arm/arm_linker.rs | 5 +- crates/fbuild-build/src/lib.rs | 2 + crates/fbuild-build/src/linker.rs | 18 +- crates/fbuild-build/src/nrf52/nrf52_linker.rs | 5 +- crates/fbuild-build/src/parallel.rs | 9 +- crates/fbuild-build/src/pipeline.rs | 63 +++- .../src/renesas/renesas_linker.rs | 5 +- crates/fbuild-build/src/sam/sam_linker.rs | 12 +- crates/fbuild-build/src/script_runtime.rs | 357 ++++++++++++++++++ .../src/script_runtime_harness.py | 274 ++++++++++++++ .../fbuild-build/src/silabs/silabs_linker.rs | 5 +- .../fbuild-build/src/teensy/teensy_linker.rs | 5 +- crates/fbuild-config/src/ini_parser.rs | 53 +++ 21 files changed, 1114 insertions(+), 51 deletions(-) create mode 100644 crates/fbuild-build/src/flag_overlay.rs create mode 100644 crates/fbuild-build/src/script_runtime.rs create mode 100644 crates/fbuild-build/src/script_runtime_harness.py diff --git a/crates/fbuild-build/Cargo.toml b/crates/fbuild-build/Cargo.toml index 9922f4ee..da1f6146 100644 --- a/crates/fbuild-build/Cargo.toml +++ b/crates/fbuild-build/Cargo.toml @@ -20,6 +20,6 @@ async-trait = { workspace = true } regex = { workspace = true } walkdir = { workspace = true } sha2 = { workspace = true } +tempfile = { workspace = true } [dev-dependencies] -tempfile = { workspace = true } diff --git a/crates/fbuild-build/src/avr/avr_linker.rs b/crates/fbuild-build/src/avr/avr_linker.rs index 9ac0f732..3d37bbe8 100644 --- a/crates/fbuild-build/src/avr/avr_linker.rs +++ b/crates/fbuild-build/src/avr/avr_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::AvrMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// AVR-specific linker using avr-gcc (link driver), avr-ar, avr-objcopy, avr-size. pub struct AvrLinker { @@ -64,6 +64,7 @@ impl Linker for AvrLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -80,6 +81,7 @@ impl Linker for AvrLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend(["-o".to_string(), elf_path.to_string_lossy().to_string()]); @@ -97,6 +99,7 @@ impl Linker for AvrLinker { // Group for circular deps + libraries from config args.push("-Wl,--start-group".to_string()); args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); args.push("-Wl,--end-group".to_string()); if self.verbose { diff --git a/crates/fbuild-build/src/ch32v/ch32v_linker.rs b/crates/fbuild-build/src/ch32v/ch32v_linker.rs index 2d6b591c..55850031 100644 --- a/crates/fbuild-build/src/ch32v/ch32v_linker.rs +++ b/crates/fbuild-build/src/ch32v/ch32v_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::Ch32vMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// CH32V-specific linker using riscv-none-elf-gcc (link driver), ar, objcopy, size. pub struct Ch32vLinker { @@ -64,6 +64,7 @@ impl Linker for Ch32vLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -77,6 +78,7 @@ impl Linker for Ch32vLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend([ format!("-T{}", self.linker_script_path.display()), @@ -96,6 +98,7 @@ impl Linker for Ch32vLinker { // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-build/src/compile_database.rs b/crates/fbuild-build/src/compile_database.rs index 8516e63a..eaf580ee 100644 --- a/crates/fbuild-build/src/compile_database.rs +++ b/crates/fbuild-build/src/compile_database.rs @@ -8,6 +8,8 @@ use std::path::{Path, PathBuf}; use fbuild_core::Result; +use crate::flag_overlay::LanguageExtraFlags; + /// A single entry in the compile database. #[derive(Debug, Clone, serde::Serialize)] pub struct CompileEntry { @@ -355,7 +357,7 @@ pub fn generate_entries( c_flags: &[String], cpp_flags: &[String], include_flags: &[String], - extra_flags: &[String], + extra_flags: &LanguageExtraFlags, sources: &[PathBuf], build_dir: &Path, project_dir: &Path, @@ -377,13 +379,15 @@ pub fn generate_entries( }; let obj = crate::compiler::CompilerBase::object_path(source, build_dir); + let source_extra_flags = extra_flags.for_source(source); - let mut arguments = - Vec::with_capacity(1 + flags.len() + include_flags.len() + extra_flags.len() + 4); + let mut arguments = Vec::with_capacity( + 1 + flags.len() + include_flags.len() + source_extra_flags.len() + 4, + ); arguments.push(compiler.to_string_lossy().to_string()); arguments.extend(flags.iter().cloned()); arguments.extend(include_flags.iter().cloned()); - arguments.extend(extra_flags.iter().cloned()); + arguments.extend(source_extra_flags); arguments.push("-c".to_string()); arguments.push(source.to_string_lossy().to_string()); arguments.push("-o".to_string()); @@ -403,6 +407,35 @@ pub fn generate_entries( mod tests { use super::*; + fn generate_entries( + gcc_path: &Path, + gxx_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + include_flags: &[String], + extra_flags: &[String], + sources: &[PathBuf], + build_dir: &Path, + project_dir: &Path, + ) -> Vec { + super::generate_entries( + gcc_path, + gxx_path, + c_flags, + cpp_flags, + include_flags, + &LanguageExtraFlags { + common: extra_flags.to_vec(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + sources, + build_dir, + project_dir, + ) + } + // --- Serialization tests --- #[test] diff --git a/crates/fbuild-build/src/esp32/esp32_linker.rs b/crates/fbuild-build/src/esp32/esp32_linker.rs index 6e3fcdbe..27839a37 100644 --- a/crates/fbuild-build/src/esp32/esp32_linker.rs +++ b/crates/fbuild-build/src/esp32/esp32_linker.rs @@ -16,7 +16,7 @@ use fbuild_core::{BuildProfile, Result, SizeInfo}; use crate::build_fingerprint::{ load_json, save_json, BinArtifactCache, FileStamp, SizeArtifactCache, BUILD_FINGERPRINT_VERSION, }; -use crate::linker::{Linker, LinkerScripts}; +use crate::linker::{LinkExtraArgs, Linker, LinkerScripts}; use super::mcu_config::Esp32McuConfig; @@ -241,6 +241,7 @@ impl Linker for Esp32Linker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -252,6 +253,7 @@ impl Linker for Esp32Linker { // Linker flags (from SDK flags/ld_flags or MCU config fallback) link_args.extend(self.linker_flags()); + link_args.extend(extra.flags.iter().cloned()); // Linker scripts (search dirs + script names from SDK) link_args.extend(self.linker_scripts.to_args()); @@ -277,6 +279,7 @@ impl Linker for Esp32Linker { // SDK precompiled libraries (ordered flags from flags/ld_libs) link_args.extend(self.sdk_lib_flags.clone()); + link_args.extend(extra.libs.iter().cloned()); link_args.push("-Wl,--end-group".to_string()); diff --git a/crates/fbuild-build/src/esp32/orchestrator.rs b/crates/fbuild-build/src/esp32/orchestrator.rs index 41c767f0..0dc4d7e9 100644 --- a/crates/fbuild-build/src/esp32/orchestrator.rs +++ b/crates/fbuild-build/src/esp32/orchestrator.rs @@ -29,6 +29,7 @@ use crate::build_fingerprint::{ hash_watch_set_stamps, load_json, normalize_path, save_json, stable_hash_json, PersistedBuildFingerprint, BUILD_FINGERPRINT_VERSION, }; +use crate::flag_overlay::LanguageExtraFlags; use crate::linker::LinkerScripts; use crate::zccache::FingerprintWatch; @@ -503,6 +504,26 @@ impl BuildOrchestrator for Esp32Orchestrator { let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; user_build_flags.extend(params.extra_build_flags.clone()); user_flags.extend(user_build_flags.clone()); + let user_overlay = LanguageExtraFlags { + common: user_flags + .iter() + .cloned() + .chain(ctx.global_compile_overlay.common.iter().cloned()) + .collect(), + c: ctx.global_compile_overlay.c.clone(), + cxx: ctx.global_compile_overlay.cxx.clone(), + asm: ctx.global_compile_overlay.asm.clone(), + }; + let src_overlay = LanguageExtraFlags::combined(&[ + &user_overlay, + &LanguageExtraFlags { + common: ctx.src_flags.clone(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + &ctx.project_compile_overlay, + ]); // Emit a warning if CDC on boot is effectively enabled (may cause Serial to block // when no USB host is connected). @@ -533,8 +554,9 @@ impl BuildOrchestrator for Esp32Orchestrator { ); // Apply user build_flags to library compilation (matching PlatformIO behavior). // User flags like -std=gnu++2a replace the MCU config's -std=gnu++2b. - let c_flags = apply_user_flags(&temp_compiler.c_flags(), &user_flags); - let cpp_flags = apply_user_flags(&temp_compiler.cpp_flags(), &user_flags); + let c_flags = apply_overlay_flags(&temp_compiler.c_flags(), &user_overlay, "dummy.c"); + let cpp_flags = + apply_overlay_flags(&temp_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); let jobs = crate::parallel::effective_jobs(params.jobs); // Use gcc-ar for LTO archives so the linker-plugin index is written. @@ -594,8 +616,9 @@ impl BuildOrchestrator for Esp32Orchestrator { params.verbose, build_dir.join("tmp"), ); - let p_c_flags = apply_user_flags(&p_compiler.c_flags(), &user_flags); - let p_cpp_flags = apply_user_flags(&p_compiler.cpp_flags(), &user_flags); + let p_c_flags = apply_overlay_flags(&p_compiler.c_flags(), &src_overlay, "dummy.c"); + let p_cpp_flags = + apply_overlay_flags(&p_compiler.cpp_flags(), &src_overlay, "dummy.cpp"); // Collect lib/* names so the helper can detect collisions with project-as-library. let mut existing_lib_names = std::collections::HashSet::new(); @@ -677,8 +700,10 @@ impl BuildOrchestrator for Esp32Orchestrator { params.verbose, build_dir.join("tmp"), ); - let fw_c_flags = apply_user_flags(&fw_compiler.c_flags(), &user_flags); - let fw_cpp_flags = apply_user_flags(&fw_compiler.cpp_flags(), &user_flags); + let fw_c_flags = + apply_overlay_flags(&fw_compiler.c_flags(), &user_overlay, "dummy.c"); + let fw_cpp_flags = + apply_overlay_flags(&fw_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); let fw_signature = framework_signature(&include_dirs, &fw_c_flags, &fw_cpp_flags); let mut fw_lib_count = 0; @@ -836,10 +861,6 @@ impl BuildOrchestrator for Esp32Orchestrator { all_core_sources.extend(sources.core_sources.iter().cloned()); all_core_sources.extend(sources.variant_sources.iter().cloned()); - let src_flags = ctx.config.get_build_src_flags(¶ms.env_name)?; - let all_src_flags: Vec = - user_flags.iter().chain(src_flags.iter()).cloned().collect(); - // Precompute values needed for compile_commands.json in both paths let include_flags = compiler.base.build_include_flags(); let arch = if mcu_config.is_xtensa() { @@ -856,8 +877,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &compiler.c_flags(), &compiler.cpp_flags(), &include_flags, - &user_flags, - &all_src_flags, + &user_overlay, + &src_overlay, &all_core_sources, &sources.sketch_sources, core_build_dir, @@ -889,7 +910,7 @@ impl BuildOrchestrator for Esp32Orchestrator { &compiler, &all_core_sources, core_build_dir, - &user_flags, + &user_overlay, jobs, Some(&build_log_mutex), )?; @@ -899,7 +920,7 @@ impl BuildOrchestrator for Esp32Orchestrator { &compiler, &sources.sketch_sources, src_build_dir, - &all_src_flags, + &src_overlay, jobs, Some(&build_log_mutex), )?; @@ -947,8 +968,10 @@ impl BuildOrchestrator for Esp32Orchestrator { // Use gcc-ar for LTO archives so the linker-plugin index is written. let local_ar_path = toolchain.get_ar_path(); let local_gcc_ar_path = toolchain.get_gcc_ar_path(); - let local_c_flags = apply_user_flags(&compiler.c_flags(), &all_src_flags); - let local_cpp_flags = apply_user_flags(&compiler.cpp_flags(), &all_src_flags); + let local_c_flags = + apply_overlay_flags(&compiler.c_flags(), &src_overlay, "dummy.c"); + let local_cpp_flags = + apply_overlay_flags(&compiler.cpp_flags(), &src_overlay, "dummy.cpp"); let local_lib_ar_path = crate::pipeline::pick_archiver( &local_ar_path, &local_gcc_ar_path, @@ -1017,8 +1040,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &compiler.c_flags(), &compiler.cpp_flags(), &include_flags, - &user_flags, - &all_src_flags, + &user_overlay, + &src_overlay, &all_core_sources, &sources.sketch_sources, core_build_dir, @@ -1055,6 +1078,10 @@ impl BuildOrchestrator for Esp32Orchestrator { &sketch_objects, &all_archives, build_dir, + &crate::linker::LinkExtraArgs { + flags: ctx.overlay_link_flags.clone(), + libs: ctx.overlay_link_libs.clone(), + }, params.symbol_analysis, )?; @@ -1275,6 +1302,14 @@ fn apply_user_flags(base_flags: &[String], user_flags: &[String]) -> Vec result } +fn apply_overlay_flags( + base_flags: &[String], + overlay: &LanguageExtraFlags, + probe_name: &str, +) -> Vec { + apply_user_flags(base_flags, &overlay.for_source(Path::new(probe_name))) +} + fn define_flag_name(flag: &str) -> Option<&str> { let define = flag.strip_prefix("-D")?; let name = define diff --git a/crates/fbuild-build/src/esp8266/esp8266_linker.rs b/crates/fbuild-build/src/esp8266/esp8266_linker.rs index 37905a40..ce665b47 100644 --- a/crates/fbuild-build/src/esp8266/esp8266_linker.rs +++ b/crates/fbuild-build/src/esp8266/esp8266_linker.rs @@ -14,7 +14,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::Esp8266McuConfig; -use crate::linker::{Linker, LinkerScripts}; +use crate::linker::{LinkExtraArgs, Linker, LinkerScripts}; /// ESP8266-specific linker using Xtensa LX106 GCC as the link driver. pub struct Esp8266Linker { @@ -140,6 +140,7 @@ impl Linker for Esp8266Linker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; @@ -184,6 +185,7 @@ impl Linker for Esp8266Linker { if let Some(profile) = self.mcu_config.profiles.get(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); // Build output dir — contains generated local.eagle.app.v6.common.ld args.push(format!("-L{}", output_dir.to_string_lossy())); @@ -210,6 +212,7 @@ impl Linker for Esp8266Linker { args.push(archive.to_string_lossy().to_string()); } args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); args.push("-Wl,--end-group".to_string()); if self.verbose { diff --git a/crates/fbuild-build/src/flag_overlay.rs b/crates/fbuild-build/src/flag_overlay.rs new file mode 100644 index 00000000..fde30d0c --- /dev/null +++ b/crates/fbuild-build/src/flag_overlay.rs @@ -0,0 +1,218 @@ +use std::path::{Path, PathBuf}; + +/// Language-aware extra compiler flags. +/// +/// PlatformIO scripts can mutate common compile scopes (`CCFLAGS`, +/// `CPPDEFINES`, `CPPPATH`) plus language-specific scopes (`CFLAGS`, +/// `CXXFLAGS`, `ASFLAGS`). Native fbuild needs to preserve that separation +/// when replaying the effective build state. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LanguageExtraFlags { + pub common: Vec, + pub c: Vec, + pub cxx: Vec, + pub asm: Vec, +} + +impl LanguageExtraFlags { + pub fn is_empty(&self) -> bool { + self.common.is_empty() && self.c.is_empty() && self.cxx.is_empty() && self.asm.is_empty() + } + + pub fn extend(&mut self, other: &Self) { + self.common.extend(other.common.iter().cloned()); + self.c.extend(other.c.iter().cloned()); + self.cxx.extend(other.cxx.iter().cloned()); + self.asm.extend(other.asm.iter().cloned()); + } + + pub fn combined(parts: &[&Self]) -> Self { + let mut merged = Self::default(); + for part in parts { + merged.extend(part); + } + merged + } + + /// Effective extra flags for a source file path. + pub fn for_source(&self, source: &Path) -> Vec { + let mut flags = self.common.clone(); + let ext = source + .extension() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + match ext.as_str() { + "c" => flags.extend(self.c.iter().cloned()), + "s" | "sx" | "asm" | "spp" => flags.extend(self.asm.iter().cloned()), + _ => flags.extend(self.cxx.iter().cloned()), + } + flags + } + + /// Flatten all scopes for diagnostics / legacy code paths. + pub fn flatten(&self) -> Vec { + let mut flags = self.common.clone(); + flags.extend(self.c.iter().cloned()); + flags.extend(self.cxx.iter().cloned()); + flags.extend(self.asm.iter().cloned()); + flags + } +} + +/// Link-time additions resolved from script mutations. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LinkExtraFlags { + pub flags: Vec, + pub libs: Vec, +} + +impl LinkExtraFlags { + pub fn is_empty(&self) -> bool { + self.flags.is_empty() && self.libs.is_empty() + } + + pub fn extend(&mut self, other: &Self) { + self.flags.extend(other.flags.iter().cloned()); + self.libs.extend(other.libs.iter().cloned()); + } +} + +/// Native replayable build overlay extracted from extra scripts. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BuildOverlay { + /// Mutations that apply to the global construction environment (`env`). + pub global_compile: LanguageExtraFlags, + /// Mutations that apply only to project sources (`projenv`). + pub project_compile: LanguageExtraFlags, + /// Additional link-time arguments. + pub link: LinkExtraFlags, + /// User-facing notes emitted by the runtime (e.g. ignored no-op actions). + pub notes: Vec, +} + +impl BuildOverlay { + pub fn is_empty(&self) -> bool { + self.global_compile.is_empty() + && self.project_compile.is_empty() + && self.link.is_empty() + && self.notes.is_empty() + } +} + +/// Serializable form returned by the Python script runtime. +#[derive(Debug, Clone, serde::Deserialize)] +pub(crate) struct ScriptRuntimeResult { + pub env: ScriptScopeState, + pub projenv: ScriptScopeState, + #[serde(default)] + pub notes: Vec, + #[serde(default)] + pub unsupported: Vec, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +pub(crate) struct ScriptScopeState { + #[serde(default)] + pub cppdefines: Vec, + #[serde(default)] + pub cpppath: Vec, + #[serde(default)] + pub ccflags: Vec, + #[serde(default)] + pub cflags: Vec, + #[serde(default)] + pub cxxflags: Vec, + #[serde(default)] + pub asflags: Vec, + #[serde(default)] + pub linkflags: Vec, + #[serde(default)] + pub libpath: Vec, + #[serde(default)] + pub libs: Vec, +} + +pub(crate) fn values_to_args( + values: &[serde_json::Value], + project_dir: &Path, +) -> fbuild_core::Result> { + let mut args = Vec::new(); + for value in values { + append_value_args(value, project_dir, &mut args)?; + } + Ok(args) +} + +fn append_value_args( + value: &serde_json::Value, + project_dir: &Path, + args: &mut Vec, +) -> fbuild_core::Result<()> { + match value { + serde_json::Value::String(s) => args.push(s.clone()), + serde_json::Value::Number(n) => args.push(n.to_string()), + serde_json::Value::Bool(b) => args.push(if *b { "1" } else { "0" }.to_string()), + serde_json::Value::Array(items) => { + for item in items { + append_value_args(item, project_dir, args)?; + } + } + serde_json::Value::Object(map) => { + if let Some(kind) = map.get("kind").and_then(|v| v.as_str()) { + match kind { + "path" => { + let raw = map.get("value").and_then(|v| v.as_str()).ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "invalid script runtime path entry".to_string(), + ) + })?; + let path = absolutize_if_relative(project_dir, raw); + args.push(path.to_string_lossy().to_string()); + } + "kv" => { + let key = map.get("key").and_then(|v| v.as_str()).ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "invalid script runtime kv entry".to_string(), + ) + })?; + let value = map.get("value").ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "invalid script runtime kv value".to_string(), + ) + })?; + args.push(match value { + serde_json::Value::String(s) => format!("{key}={s}"), + serde_json::Value::Number(n) => format!("{key}={n}"), + serde_json::Value::Bool(b) => { + format!("{key}={}", if *b { "1" } else { "0" }) + } + _ => { + return Err(fbuild_core::FbuildError::BuildFailed( + "unsupported script runtime kv value".to_string(), + )) + } + }); + } + _ => { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "unsupported script runtime entry kind '{}'", + kind + ))) + } + } + } + } + serde_json::Value::Null => {} + } + Ok(()) +} + +pub(crate) fn absolutize_if_relative(project_dir: &Path, raw: &str) -> PathBuf { + let path = PathBuf::from(raw); + if path.is_absolute() { + path + } else { + project_dir.join(path) + } +} diff --git a/crates/fbuild-build/src/generic_arm/arm_linker.rs b/crates/fbuild-build/src/generic_arm/arm_linker.rs index 44fede09..afcf5d0a 100644 --- a/crates/fbuild-build/src/generic_arm/arm_linker.rs +++ b/crates/fbuild-build/src/generic_arm/arm_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::ArmMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// Generic ARM linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct ArmLinker { @@ -64,6 +64,7 @@ impl Linker for ArmLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -77,6 +78,7 @@ impl Linker for ArmLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend([ format!("-T{}", self.linker_script_path.display()), @@ -96,6 +98,7 @@ impl Linker for ArmLinker { // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-build/src/lib.rs b/crates/fbuild-build/src/lib.rs index 91da35b9..825667b0 100644 --- a/crates/fbuild-build/src/lib.rs +++ b/crates/fbuild-build/src/lib.rs @@ -13,6 +13,7 @@ pub mod compile_database; pub mod compiler; pub mod esp32; pub mod esp8266; +pub mod flag_overlay; pub mod generic_arm; pub mod linker; pub mod nrf52; @@ -21,6 +22,7 @@ pub mod pipeline; pub mod renesas; pub mod rp2040; pub mod sam; +pub mod script_runtime; pub mod silabs; pub mod source_scanner; pub mod stm32; diff --git a/crates/fbuild-build/src/linker.rs b/crates/fbuild-build/src/linker.rs index 46d64190..8ae47df4 100644 --- a/crates/fbuild-build/src/linker.rs +++ b/crates/fbuild-build/src/linker.rs @@ -19,6 +19,13 @@ pub struct LinkResult { pub stderr: String, } +/// Additional link arguments resolved outside the platform linker config. +#[derive(Debug, Clone, Default)] +pub struct LinkExtraArgs { + pub flags: Vec, + pub libs: Vec, +} + /// Check whether `elf_path` is newer than every file in `inputs`. /// /// Returns `true` when the ELF can be reused (no input was modified since @@ -46,7 +53,13 @@ pub trait Linker: Send + Sync { fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()>; /// Link objects and archives into an ELF binary. - fn link(&self, objects: &[PathBuf], archives: &[PathBuf], output: &Path) -> Result; + fn link( + &self, + objects: &[PathBuf], + archives: &[PathBuf], + output: &Path, + extra: &LinkExtraArgs, + ) -> Result; /// Convert ELF to firmware format (.hex, .bin, etc.). fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result; @@ -69,6 +82,7 @@ pub trait Linker: Send + Sync { sketch_objects: &[PathBuf], core_objects: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, symbol_analysis: bool, ) -> Result { let candidate_elf = output_dir.join("firmware.elf"); @@ -106,7 +120,7 @@ pub trait Linker: Send + Sync { // Pass core objects directly to linker (not archived) for LTO compatibility. // With LTO + archives, the linker can't see symbols across TUs properly. - let elf_path = self.link(sketch_objects, core_objects, output_dir)?; + let elf_path = self.link(sketch_objects, core_objects, output_dir, extra)?; // Convert let firmware_path = self.convert_firmware(&elf_path, output_dir)?; diff --git a/crates/fbuild-build/src/nrf52/nrf52_linker.rs b/crates/fbuild-build/src/nrf52/nrf52_linker.rs index 6024794c..6e658497 100644 --- a/crates/fbuild-build/src/nrf52/nrf52_linker.rs +++ b/crates/fbuild-build/src/nrf52/nrf52_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::Nrf52McuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// NRF52-specific linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct Nrf52Linker { @@ -67,6 +67,7 @@ impl Linker for Nrf52Linker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -80,6 +81,7 @@ impl Linker for Nrf52Linker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); // Linker search dirs (for INCLUDE directives in linker scripts) for dir in &self.linker_search_dirs { @@ -104,6 +106,7 @@ impl Linker for Nrf52Linker { // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-build/src/parallel.rs b/crates/fbuild-build/src/parallel.rs index 3883786e..7c8c1fc5 100644 --- a/crates/fbuild-build/src/parallel.rs +++ b/crates/fbuild-build/src/parallel.rs @@ -10,6 +10,7 @@ use std::sync::Mutex; use fbuild_core::{BuildLog, FbuildError, Result}; use crate::compiler::{Compiler, CompilerBase}; +use crate::flag_overlay::LanguageExtraFlags; /// Default job count: num_cpus * 2. pub fn default_jobs() -> usize { @@ -42,7 +43,7 @@ pub fn compile_sources_parallel( compiler: &dyn Compiler, sources: &[PathBuf], build_dir: &Path, - extra_flags: &[String], + extra_flags: &LanguageExtraFlags, jobs: usize, build_log: Option<&Mutex>, ) -> Result { @@ -52,7 +53,8 @@ pub fn compile_sources_parallel( for source in sources { let obj = CompilerBase::object_path(source, build_dir); - let signature = compiler.rebuild_signature(source, extra_flags); + let source_flags = extra_flags.for_source(source); + let signature = compiler.rebuild_signature(source, &source_flags); if CompilerBase::needs_rebuild_with_signature(source, &obj, Some(&signature)) { work.push((source.clone(), obj.clone())); } @@ -92,7 +94,8 @@ pub fn compile_sources_parallel( None => break, }; - match compiler.compile(&source, &obj, extra_flags) { + let source_flags = extra_flags.for_source(&source); + match compiler.compile(&source, &obj, &source_flags) { Ok(result) if result.success => { let stderr = result.stderr.trim().to_string(); if !stderr.is_empty() { diff --git a/crates/fbuild-build/src/pipeline.rs b/crates/fbuild-build/src/pipeline.rs index 7cc233e1..ccd1ce30 100644 --- a/crates/fbuild-build/src/pipeline.rs +++ b/crates/fbuild-build/src/pipeline.rs @@ -10,6 +10,7 @@ use fbuild_core::{BuildLog, Result}; use crate::compile_database::{self, CompileDatabase, TargetArchitecture}; use crate::compiler::Compiler; +use crate::flag_overlay::LanguageExtraFlags; use crate::linker::LinkResult; use crate::source_scanner::SourceCollection; use crate::{BuildParams, BuildResult}; @@ -29,6 +30,10 @@ pub struct BuildContext { pub user_flags: Vec, pub src_flags: Vec, pub all_src_flags: Vec, + pub global_compile_overlay: LanguageExtraFlags, + pub project_compile_overlay: LanguageExtraFlags, + pub overlay_link_flags: Vec, + pub overlay_link_libs: Vec, } impl BuildContext { @@ -49,6 +54,8 @@ impl BuildContext { let config = fbuild_config::PlatformIOConfig::from_path_with_overrides(&ini_path, pio_overrides)?; let env_config = config.get_env_config(env_name)?; + let overlay = + crate::script_runtime::resolve_extra_script_overlay(project_dir, env_name, &config)?; // 2. Load board config let board_id = env_config.get("board").ok_or_else(|| { @@ -75,6 +82,9 @@ impl BuildContext { board.max_flash, board.max_ram, ); + for note in &overlay.notes { + build_log.push(format!("extra_scripts: {}", note)); + } // 4. Setup build directories let cache = fbuild_packages::Cache::new(project_dir); @@ -121,6 +131,10 @@ impl BuildContext { user_flags, src_flags, all_src_flags, + global_compile_overlay: overlay.global_compile, + project_compile_overlay: overlay.project_compile, + overlay_link_flags: overlay.link.flags, + overlay_link_libs: overlay.link.libs, }) } } @@ -185,7 +199,7 @@ pub fn compile_sources( compiler: &dyn Compiler, sources: &[PathBuf], build_dir: &Path, - extra_flags: &[String], + extra_flags: &LanguageExtraFlags, jobs: usize, build_log: &std::sync::Mutex, ) -> Result> { @@ -215,7 +229,7 @@ pub fn compile_local_libraries( compiler: &dyn Compiler, project_dir: &Path, build_dir: &Path, - extra_flags: &[String], + extra_flags: &LanguageExtraFlags, jobs: usize, build_log: &std::sync::Mutex, ) -> Result> { @@ -287,8 +301,8 @@ pub fn generate_compile_db( c_flags: &[String], cpp_flags: &[String], include_flags: &[String], - user_flags: &[String], - all_src_flags: &[String], + user_flags: &LanguageExtraFlags, + all_src_flags: &LanguageExtraFlags, core_sources: &[PathBuf], sketch_sources: &[PathBuf], core_build_dir: &Path, @@ -461,6 +475,27 @@ pub fn run_sequential_build_with_libs( .chain(sources.variant_sources.iter()) .cloned() .collect(); + let user_overlay = LanguageExtraFlags { + common: ctx + .user_flags + .iter() + .cloned() + .chain(ctx.global_compile_overlay.common.iter().cloned()) + .collect(), + c: ctx.global_compile_overlay.c.clone(), + cxx: ctx.global_compile_overlay.cxx.clone(), + asm: ctx.global_compile_overlay.asm.clone(), + }; + let src_overlay = LanguageExtraFlags::combined(&[ + &user_overlay, + &LanguageExtraFlags { + common: ctx.src_flags.clone(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + &ctx.project_compile_overlay, + ]); // compiledb_only: generate compile_commands.json without compiling if params.compiledb_only { @@ -470,8 +505,8 @@ pub fn run_sequential_build_with_libs( &compiler.c_flags(), &compiler.cpp_flags(), &[], - &ctx.user_flags, - &ctx.all_src_flags, + &user_overlay, + &src_overlay, &core_and_variant, &sources.sketch_sources, &ctx.core_build_dir, @@ -506,7 +541,7 @@ pub fn run_sequential_build_with_libs( compiler, &sources.core_sources, &ctx.core_build_dir, - &ctx.user_flags, + &user_overlay, jobs, &build_log_mutex, )?; @@ -514,7 +549,7 @@ pub fn run_sequential_build_with_libs( compiler, &sources.variant_sources, &ctx.core_build_dir, - &ctx.user_flags, + &user_overlay, jobs, &build_log_mutex, )?; @@ -525,7 +560,7 @@ pub fn run_sequential_build_with_libs( compiler, &sources.sketch_sources, &ctx.src_build_dir, - &ctx.all_src_flags, + &src_overlay, jobs, &build_log_mutex, )?; @@ -535,7 +570,7 @@ pub fn run_sequential_build_with_libs( compiler, ¶ms.project_dir, &ctx.build_dir, - &ctx.all_src_flags, + &src_overlay, jobs, &build_log_mutex, )?; @@ -581,8 +616,8 @@ pub fn run_sequential_build_with_libs( &compiler.c_flags(), &compiler.cpp_flags(), &[], - &ctx.user_flags, - &ctx.all_src_flags, + &user_overlay, + &src_overlay, &core_and_variant, &sources.sketch_sources, &ctx.core_build_dir, @@ -605,6 +640,10 @@ pub fn run_sequential_build_with_libs( &sketch_objects, &core_objects, &ctx.build_dir, + &crate::linker::LinkExtraArgs { + flags: ctx.overlay_link_flags.clone(), + libs: ctx.overlay_link_libs.clone(), + }, params.symbol_analysis, )?; diff --git a/crates/fbuild-build/src/renesas/renesas_linker.rs b/crates/fbuild-build/src/renesas/renesas_linker.rs index 068b95f8..d0f5eea9 100644 --- a/crates/fbuild-build/src/renesas/renesas_linker.rs +++ b/crates/fbuild-build/src/renesas/renesas_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::RenesasMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// Renesas-specific linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct RenesasLinker { @@ -64,6 +64,7 @@ impl Linker for RenesasLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -77,6 +78,7 @@ impl Linker for RenesasLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); // Add variant directory to linker search path so INCLUDE directives // in fsp.ld can find memory_regions.ld @@ -109,6 +111,7 @@ impl Linker for RenesasLinker { // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-build/src/sam/sam_linker.rs b/crates/fbuild-build/src/sam/sam_linker.rs index d21508d9..b69eecf9 100644 --- a/crates/fbuild-build/src/sam/sam_linker.rs +++ b/crates/fbuild-build/src/sam/sam_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::SamMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// SAM-specific linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct SamLinker { @@ -78,6 +78,7 @@ impl Linker for SamLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -91,6 +92,7 @@ impl Linker for SamLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend([ format!("-T{}", self.linker_script_path.display()), @@ -115,11 +117,17 @@ impl Linker for SamLinker { // Extra libraries (e.g. variant system lib) for lib in &self.extra_libs { - args.push(format!("-l{}", lib)); + if lib.starts_with("-l") || lib.contains(std::path::MAIN_SEPARATOR) || lib.contains('/') + { + args.push(lib.clone()); + } else { + args.push(format!("-l{}", lib)); + } } // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-build/src/script_runtime.rs b/crates/fbuild-build/src/script_runtime.rs new file mode 100644 index 00000000..5a94cfb1 --- /dev/null +++ b/crates/fbuild-build/src/script_runtime.rs @@ -0,0 +1,357 @@ +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use crate::flag_overlay::{ + absolutize_if_relative, values_to_args, BuildOverlay, LanguageExtraFlags, LinkExtraFlags, + ScriptRuntimeResult, ScriptScopeState, +}; + +const HARNESS: &str = include_str!("script_runtime_harness.py"); + +#[derive(Debug, serde::Serialize)] +struct ScriptRuntimeInput<'a> { + project_dir: &'a str, + env_name: &'a str, + extra_scripts: &'a [String], + project_options: &'a HashMap, +} + +pub fn resolve_extra_script_overlay( + project_dir: &Path, + env_name: &str, + config: &fbuild_config::PlatformIOConfig, +) -> fbuild_core::Result { + let extra_scripts = config.get_extra_scripts(env_name)?; + if extra_scripts.is_empty() { + return Ok(BuildOverlay::default()); + } + + let project_options = config.get_env_config(env_name)?; + let input = ScriptRuntimeInput { + project_dir: &project_dir.to_string_lossy(), + env_name, + extra_scripts: &extra_scripts, + project_options, + }; + + let python = find_python().ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "extra_scripts detected but no Python interpreter was found; \ + install Python or use --platformio" + .to_string(), + ) + })?; + + let temp_dir = tempfile::tempdir().map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to create temporary directory for extra_scripts runtime: {}", + e + )) + })?; + let harness_path = temp_dir.path().join("fbuild_extra_scripts_runtime.py"); + let input_path = temp_dir.path().join("input.json"); + std::fs::write(&harness_path, HARNESS).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to write extra_scripts harness: {}", + e + )) + })?; + std::fs::write( + &input_path, + serde_json::to_vec_pretty(&input).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to serialize extra_scripts runtime input: {}", + e + )) + })?, + ) + .map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to write extra_scripts runtime input: {}", + e + )) + })?; + + let mut command = Command::new(&python[0]); + if python.len() > 1 { + command.args(&python[1..]); + } + command + .arg(&harness_path) + .arg(&input_path) + .current_dir(project_dir) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let output = command.output().map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to run extra_scripts runtime via '{}': {}", + python.join(" "), + e + )) + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "extra_scripts runtime failed: {}\nRecommendation: use --platformio for this project.", + if stderr.is_empty() { + format!("process exited with status {}", output.status) + } else { + stderr + } + ))); + } + + let runtime: ScriptRuntimeResult = serde_json::from_slice(&output.stdout).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to parse extra_scripts runtime output: {}", + e + )) + })?; + + if !runtime.unsupported.is_empty() { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "unsupported extra_scripts operations detected: {}\nRecommendation: use --platformio for this project.", + runtime.unsupported.join("; ") + ))); + } + + let mut overlay = BuildOverlay { + global_compile: scope_to_compile_overlay(project_dir, &runtime.env)?, + project_compile: scope_to_compile_overlay(project_dir, &runtime.projenv)?, + link: scope_to_link_overlay(project_dir, &runtime.env)?, + notes: runtime.notes, + }; + // Project-only scope can also contribute link flags in user scripts. + overlay + .link + .extend(&scope_to_link_overlay(project_dir, &runtime.projenv)?); + Ok(overlay) +} + +fn scope_to_compile_overlay( + project_dir: &Path, + scope: &ScriptScopeState, +) -> fbuild_core::Result { + let mut common = cppdefines_to_flags(&scope.cppdefines)?; + common.extend(cpppath_to_flags(project_dir, &scope.cpppath)?); + common.extend(values_to_args(&scope.ccflags, project_dir)?); + Ok(LanguageExtraFlags { + common, + c: values_to_args(&scope.cflags, project_dir)?, + cxx: values_to_args(&scope.cxxflags, project_dir)?, + asm: values_to_args(&scope.asflags, project_dir)?, + }) +} + +fn scope_to_link_overlay( + project_dir: &Path, + scope: &ScriptScopeState, +) -> fbuild_core::Result { + let mut flags = values_to_args(&scope.linkflags, project_dir)?; + for path in &scope.libpath { + for entry in values_to_args(std::slice::from_ref(path), project_dir)? { + let resolved = absolutize_if_relative(project_dir, &entry); + flags.push(format!("-L{}", resolved.display())); + } + } + let libs = libs_to_flags(project_dir, &scope.libs)?; + Ok(LinkExtraFlags { flags, libs }) +} + +fn cpppath_to_flags( + project_dir: &Path, + values: &[serde_json::Value], +) -> fbuild_core::Result> { + let mut flags = Vec::new(); + for entry in values_to_args(values, project_dir)? { + let resolved = absolutize_if_relative(project_dir, &entry); + flags.push(format!("-I{}", resolved.display())); + } + Ok(flags) +} + +fn cppdefines_to_flags(values: &[serde_json::Value]) -> fbuild_core::Result> { + let mut flags = Vec::new(); + for value in values { + match value { + serde_json::Value::String(s) => flags.push(format!("-D{}", s)), + serde_json::Value::Object(map) => { + let kind = map.get("kind").and_then(|v| v.as_str()).ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "invalid extra_scripts CPPDEFINES entry".to_string(), + ) + })?; + if kind != "kv" { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "unsupported CPPDEFINES entry kind '{}'", + kind + ))); + } + let key = map.get("key").and_then(|v| v.as_str()).ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "missing CPPDEFINES key from script runtime".to_string(), + ) + })?; + let value = map.get("value").ok_or_else(|| { + fbuild_core::FbuildError::BuildFailed( + "missing CPPDEFINES value from script runtime".to_string(), + ) + })?; + let rendered = match value { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => { + if *b { + "1".to_string() + } else { + "0".to_string() + } + } + _ => { + return Err(fbuild_core::FbuildError::BuildFailed( + "unsupported CPPDEFINES value type".to_string(), + )) + } + }; + flags.push(format!("-D{}={}", key, rendered)); + } + _ => { + return Err(fbuild_core::FbuildError::BuildFailed( + "unsupported CPPDEFINES script runtime entry".to_string(), + )) + } + } + } + Ok(flags) +} + +fn libs_to_flags( + project_dir: &Path, + values: &[serde_json::Value], +) -> fbuild_core::Result> { + let mut flags = Vec::new(); + for value in values { + for entry in values_to_args(std::slice::from_ref(value), project_dir)? { + if entry.starts_with("-l") { + flags.push(entry); + continue; + } + let looks_like_path = entry.contains(std::path::MAIN_SEPARATOR) + || entry.contains('/') + || entry.ends_with(".a") + || entry.ends_with(".o") + || entry.ends_with(".lib"); + if looks_like_path { + let resolved = absolutize_if_relative(project_dir, &entry); + flags.push(resolved.to_string_lossy().to_string()); + } else { + flags.push(format!("-l{}", entry)); + } + } + } + Ok(flags) +} + +fn find_python() -> Option> { + let candidates: &[&[&str]] = if cfg!(windows) { + &[&["python"], &["py", "-3"]] + } else { + &[&["python3"], &["python"]] + }; + + for candidate in candidates { + let mut command = Command::new(candidate[0]); + if candidate.len() > 1 { + command.args(&candidate[1..]); + } + if let Ok(output) = command.arg("--version").output() { + if output.status.success() { + return Some(candidate.iter().map(|s| (*s).to_string()).collect()); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::flag_overlay::ScriptScopeState; + + #[test] + fn test_cppdefines_to_flags_string_and_kv() { + let flags = cppdefines_to_flags(&[ + serde_json::Value::String("FOO".to_string()), + serde_json::json!({"kind": "kv", "key": "BAR", "value": "baz"}), + serde_json::json!({"kind": "kv", "key": "COUNT", "value": 7}), + ]) + .unwrap(); + assert_eq!(flags, vec!["-DFOO", "-DBAR=baz", "-DCOUNT=7"]); + } + + #[test] + fn test_libs_to_flags_names_and_paths() { + let project_dir = Path::new("/tmp/project"); + let flags = libs_to_flags( + project_dir, + &[ + serde_json::Value::String("m".to_string()), + serde_json::Value::String("libs/foo.a".to_string()), + ], + ) + .unwrap(); + assert_eq!(flags[0], "-lm"); + assert!(flags[1].ends_with("/tmp/project/libs/foo.a")); + } + + #[test] + fn test_scope_to_compile_overlay_maps_common_and_language_flags() { + let project_dir = Path::new("/tmp/project"); + let scope = ScriptScopeState { + cppdefines: vec![ + serde_json::Value::String("FOO".to_string()), + serde_json::json!({"kind": "kv", "key": "BAR", "value": 1}), + ], + cpppath: vec![serde_json::Value::String("include".to_string())], + ccflags: vec![serde_json::Value::String("-Wall".to_string())], + cflags: vec![serde_json::Value::String("-std=c11".to_string())], + cxxflags: vec![serde_json::Value::String("-std=gnu++20".to_string())], + asflags: vec![serde_json::Value::String("-x".to_string())], + ..Default::default() + }; + + let overlay = scope_to_compile_overlay(project_dir, &scope).unwrap(); + assert!(overlay.common.contains(&"-DFOO".to_string())); + assert!(overlay.common.contains(&"-DBAR=1".to_string())); + assert!(overlay + .common + .contains(&"-I/tmp/project/include".to_string())); + assert!(overlay.common.contains(&"-Wall".to_string())); + assert_eq!(overlay.c, vec!["-std=c11"]); + assert_eq!(overlay.cxx, vec!["-std=gnu++20"]); + assert_eq!(overlay.asm, vec!["-x"]); + } + + #[test] + fn test_scope_to_link_overlay_maps_libpath_and_libs() { + let project_dir = Path::new("/tmp/project"); + let scope = ScriptScopeState { + linkflags: vec![serde_json::Value::String("-Wl,--gc-sections".to_string())], + libpath: vec![serde_json::Value::String("lib".to_string())], + libs: vec![ + serde_json::Value::String("m".to_string()), + serde_json::Value::String("archives/foo.a".to_string()), + ], + ..Default::default() + }; + + let overlay = scope_to_link_overlay(project_dir, &scope).unwrap(); + assert!(overlay.flags.contains(&"-Wl,--gc-sections".to_string())); + assert!(overlay.flags.contains(&"-L/tmp/project/lib".to_string())); + assert_eq!(overlay.libs[0], "-lm"); + assert!(overlay.libs[1].ends_with("/tmp/project/archives/foo.a")); + } +} diff --git a/crates/fbuild-build/src/script_runtime_harness.py b/crates/fbuild-build/src/script_runtime_harness.py new file mode 100644 index 00000000..ea9a7a01 --- /dev/null +++ b/crates/fbuild-build/src/script_runtime_harness.py @@ -0,0 +1,274 @@ +import inspect +import json +import os +import re +import sys + + +SUPPORTED_SCOPES = { + "CPPDEFINES", + "CPPPATH", + "CCFLAGS", + "CFLAGS", + "CXXFLAGS", + "ASFLAGS", + "LINKFLAGS", + "LIBPATH", + "LIBS", +} + +NOOP_METHODS = { + "AddPostAction", + "AddPreAction", + "AlwaysBuild", + "Alias", + "Depends", +} + + +def stringify_macro(value): + return '\\"%s\\"' % str(value).replace('"', '\\\\\\"') + + +class RuntimeFailure(Exception): + pass + + +class MockEnv: + def __init__(self, label, project_dir, env_name, project_options, notes, unsupported): + self._label = label + self._project_dir = project_dir + self._env_name = env_name + self._project_options = project_options + self._notes = notes + self._unsupported = unsupported + self._scopes = {key: [] for key in SUPPORTED_SCOPES} + self._vars = { + "PROJECT_DIR": project_dir, + "PIOENV": env_name, + } + + def export_state(self): + return { + "cppdefines": self._scopes["CPPDEFINES"], + "cpppath": self._scopes["CPPPATH"], + "ccflags": self._scopes["CCFLAGS"], + "cflags": self._scopes["CFLAGS"], + "cxxflags": self._scopes["CXXFLAGS"], + "asflags": self._scopes["ASFLAGS"], + "linkflags": self._scopes["LINKFLAGS"], + "libpath": self._scopes["LIBPATH"], + "libs": self._scopes["LIBS"], + } + + def _record_unsupported(self, message): + self._unsupported.append(message) + raise RuntimeFailure(message) + + def _normalize_path(self, value): + if value is None: + return None + resolved = self.subst(str(value)) + if os.path.isabs(resolved): + return {"kind": "path", "value": resolved} + return {"kind": "path", "value": os.path.join(self._project_dir, resolved)} + + def _normalize_cppdefine(self, value): + if isinstance(value, (list, tuple)): + if len(value) == 2 and not isinstance(value[0], (list, tuple, dict)): + return [ + { + "kind": "kv", + "key": str(value[0]), + "value": value[1], + } + ] + items = [] + for item in value: + items.extend(self._normalize_cppdefine(item)) + return items + if isinstance(value, dict): + return [value] + return [str(value)] + + def _normalize_generic(self, value): + if isinstance(value, (list, tuple)): + items = [] + for item in value: + items.extend(self._normalize_generic(item)) + return items + return [value] + + def _normalize_items(self, scope, value): + if scope == "CPPDEFINES": + return self._normalize_cppdefine(value) + if scope in ("CPPPATH", "LIBPATH"): + return [self._normalize_path(item) for item in self._normalize_generic(value)] + return [str(item) for item in self._normalize_generic(value)] + + def _apply(self, scope, items, mode): + current = self._scopes[scope] + if mode == "replace": + self._scopes[scope] = items + return + for item in items: + if mode == "append": + current.append(item) + elif mode == "prepend": + current.insert(0, item) + elif mode == "append_unique": + if item not in current: + current.append(item) + + def _mutate(self, mode, kwargs): + for scope, value in kwargs.items(): + if scope not in SUPPORTED_SCOPES: + self._record_unsupported( + f"{self._label}.{mode} on unsupported scope '{scope}'" + ) + self._apply(scope, self._normalize_items(scope, value), mode) + + def Append(self, **kwargs): + self._mutate("append", kwargs) + + def AppendUnique(self, **kwargs): + self._mutate("append_unique", kwargs) + + def Prepend(self, **kwargs): + self._mutate("prepend", kwargs) + + def Replace(self, **kwargs): + self._mutate("replace", kwargs) + + def StringifyMacro(self, value): + return stringify_macro(value) + + def subst(self, text): + text = str(text) + + def repl(match): + key = match.group(1) or match.group(2) + return str(self._vars.get(key, self._project_options.get(key, match.group(0)))) + + return re.sub(r"\$([A-Za-z_][A-Za-z0-9_]*)|\$\{([^}]+)\}", repl, text) + + def get(self, key, default=None): + return self._vars.get(key, default) + + def GetProjectOption(self, key, default=None): + return self._project_options.get(key, default) + + def __getitem__(self, key): + if key in self._vars: + return self._vars[key] + if key in self._scopes: + return self._scopes[key] + raise KeyError(key) + + def __setitem__(self, key, value): + if key in SUPPORTED_SCOPES: + self._scopes[key] = self._normalize_items(key, value) + else: + self._vars[key] = value + + def __getattr__(self, name): + if name in NOOP_METHODS: + def _noop(*args, **kwargs): + self._notes.append(f"{self._label}.{name} ignored by native extra_scripts runtime") + return None + + return _noop + self._record_unsupported(f"{self._label}.{name} is not supported") + + +def resolve_script_entry(env, raw): + if ":" in raw: + prefix, path = raw.split(":", 1) + if prefix not in ("pre", "post"): + raise RuntimeFailure(f"unsupported extra_scripts prefix '{prefix}'") + return prefix, os.path.abspath(env.subst(path)) + return "post", os.path.abspath(env.subst(raw)) + + +def execute_script(path, import_fn, project_dir): + script_dir = os.path.dirname(path) + old_sys_path = list(sys.path) + sys.path.insert(0, script_dir) + if project_dir not in sys.path: + sys.path.insert(0, project_dir) + scope = { + "__file__": path, + "__name__": "__main__", + "Import": import_fn, + "DefaultEnvironment": import_fn.default_environment, + "COMMAND_LINE_TARGETS": [], + "ARGUMENTS": {}, + } + try: + with open(path, "r", encoding="utf8") as handle: + code = compile(handle.read(), path, "exec") + exec(code, scope, scope) + finally: + sys.path[:] = old_sys_path + + +class ImportDispatcher: + def __init__(self, env, projenv, allow_projenv): + self._env = env + self._projenv = projenv + self._allow_projenv = allow_projenv + + def __call__(self, *names): + frame = inspect.currentframe().f_back + for name in names: + if name == "env": + frame.f_globals["env"] = self._env + elif name == "projenv": + if not self._allow_projenv: + raise RuntimeFailure("projenv is not available in PRE extra_scripts") + frame.f_globals["projenv"] = self._projenv + else: + raise RuntimeFailure(f"Import('{name}') is not supported") + + def default_environment(self): + return self._env + + +def main(): + if len(sys.argv) != 2: + raise RuntimeFailure("usage: harness.py ") + + with open(sys.argv[1], "r", encoding="utf8") as handle: + data = json.load(handle) + + project_dir = os.path.abspath(data["project_dir"]) + env_name = data["env_name"] + project_options = data.get("project_options", {}) + notes = [] + unsupported = [] + env = MockEnv("env", project_dir, env_name, project_options, notes, unsupported) + projenv = MockEnv("projenv", project_dir, env_name, project_options, notes, unsupported) + + script_entries = [resolve_script_entry(env, item) for item in data.get("extra_scripts", [])] + pre_scripts = [path for scope, path in script_entries if scope == "pre"] + post_scripts = [path for scope, path in script_entries if scope == "post"] + + try: + for path in pre_scripts: + execute_script(path, ImportDispatcher(env, projenv, False), project_dir) + for path in post_scripts: + execute_script(path, ImportDispatcher(env, projenv, True), project_dir) + except RuntimeFailure as exc: + unsupported.append(str(exc)) + + result = { + "env": env.export_state(), + "projenv": projenv.export_state(), + "notes": notes, + "unsupported": unsupported, + } + json.dump(result, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/crates/fbuild-build/src/silabs/silabs_linker.rs b/crates/fbuild-build/src/silabs/silabs_linker.rs index 213407ed..06c67cd7 100644 --- a/crates/fbuild-build/src/silabs/silabs_linker.rs +++ b/crates/fbuild-build/src/silabs/silabs_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::SilabsMcuConfig; -use crate::linker::Linker; +use crate::linker::{LinkExtraArgs, Linker}; /// Silicon Labs-specific linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct SilabsLinker { @@ -70,6 +70,7 @@ impl Linker for SilabsLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -83,6 +84,7 @@ impl Linker for SilabsLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend([ format!("-T{}", self.linker_script_path.display()), @@ -108,6 +110,7 @@ impl Linker for SilabsLinker { args.push("-Wl,--start-group".to_string()); args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); for archive in &self.precompiled_libs { args.push(archive.to_string_lossy().to_string()); } diff --git a/crates/fbuild-build/src/teensy/teensy_linker.rs b/crates/fbuild-build/src/teensy/teensy_linker.rs index f00d1ff0..3669a626 100644 --- a/crates/fbuild-build/src/teensy/teensy_linker.rs +++ b/crates/fbuild-build/src/teensy/teensy_linker.rs @@ -9,7 +9,7 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{BuildProfile, Result, SizeInfo}; use super::mcu_config::TeensyMcuConfig; -use crate::linker::{Linker, LinkerScripts}; +use crate::linker::{LinkExtraArgs, Linker, LinkerScripts}; /// Teensy-specific linker using arm-none-eabi-gcc (link driver), ar, objcopy, size. pub struct TeensyLinker { @@ -64,6 +64,7 @@ impl Linker for TeensyLinker { objects: &[PathBuf], archives: &[PathBuf], output_dir: &Path, + extra: &LinkExtraArgs, ) -> Result { std::fs::create_dir_all(output_dir)?; let elf_path = output_dir.join("firmware.elf"); @@ -77,6 +78,7 @@ impl Linker for TeensyLinker { if let Some(profile) = self.mcu_config.get_profile(self.profile.as_dir_name()) { args.extend(profile.link_flags.iter().cloned()); } + args.extend(extra.flags.iter().cloned()); args.extend(self.linker_scripts.to_args()); args.extend(["-o".to_string(), elf_path.to_string_lossy().to_string()]); @@ -93,6 +95,7 @@ impl Linker for TeensyLinker { // Linker libraries from config args.extend(self.mcu_config.linker_libs.iter().cloned()); + args.extend(extra.libs.iter().cloned()); if self.verbose { tracing::info!("link: {}", args.join(" ")); diff --git a/crates/fbuild-config/src/ini_parser.rs b/crates/fbuild-config/src/ini_parser.rs index 9b294a7d..dc39fd8a 100644 --- a/crates/fbuild-config/src/ini_parser.rs +++ b/crates/fbuild-config/src/ini_parser.rs @@ -161,6 +161,18 @@ impl PlatformIOConfig { } } + /// Get `extra_scripts` for an environment. + /// + /// PlatformIO treats entries without an explicit prefix as POST scripts. + /// Values may be provided comma-separated or as a multi-line list. + pub fn get_extra_scripts(&self, env_name: &str) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + match config.get("extra_scripts") { + Some(scripts) => Ok(parse_list_values(scripts)), + None => Ok(Vec::new()), + } + } + /// Get `board_build.embed_files` for an environment (binary files to embed). pub fn get_embed_files(&self, env_name: &str) -> fbuild_core::Result> { let overrides = self.get_board_overrides(env_name)?; @@ -644,6 +656,22 @@ fn parse_lib_deps(deps_str: &str) -> Vec { result } +/// Parse a generic multi-value option from a multi-line or comma-separated string. +fn parse_list_values(value: &str) -> Vec { + let mut result = Vec::new(); + + for line in value.lines() { + for item in line.split(',') { + let trimmed = item.trim(); + if !trimmed.is_empty() { + result.push(trimmed.to_string()); + } + } + } + + result +} + #[cfg(test)] mod tests { use super::*; @@ -806,6 +834,31 @@ build_flags = -D FOO -D BAR assert_eq!(flags, vec!["-DFOO", "-DBAR"]); } + #[test] + fn test_get_extra_scripts_multiline_and_comma_separated() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +extra_scripts = + pre:scripts/pre.py, scripts/post.py + post:scripts/after.py +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let scripts = config.get_extra_scripts("uno").unwrap(); + assert_eq!( + scripts, + vec![ + "pre:scripts/pre.py", + "scripts/post.py", + "post:scripts/after.py" + ] + ); + } + #[test] fn test_get_lib_deps_present() { let f = write_ini( From 5829f28c54999d060f28476b433ed2b766a2710f Mon Sep 17 00:00:00 2001 From: zackees Date: Mon, 13 Apr 2026 15:55:56 -0700 Subject: [PATCH 2/3] Allow compile database test helper arity --- crates/fbuild-build/src/compile_database.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fbuild-build/src/compile_database.rs b/crates/fbuild-build/src/compile_database.rs index eaf580ee..c19db7ae 100644 --- a/crates/fbuild-build/src/compile_database.rs +++ b/crates/fbuild-build/src/compile_database.rs @@ -407,6 +407,7 @@ pub fn generate_entries( mod tests { use super::*; + #[allow(clippy::too_many_arguments)] fn generate_entries( gcc_path: &Path, gxx_path: &Path, From 8ffc7613eea51e683827572c68ff9d065e3b5803 Mon Sep 17 00:00:00 2001 From: zackees Date: Mon, 13 Apr 2026 16:04:03 -0700 Subject: [PATCH 3/3] Normalize script runtime path tests --- crates/fbuild-build/src/script_runtime.rs | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/fbuild-build/src/script_runtime.rs b/crates/fbuild-build/src/script_runtime.rs index 5a94cfb1..30416475 100644 --- a/crates/fbuild-build/src/script_runtime.rs +++ b/crates/fbuild-build/src/script_runtime.rs @@ -304,7 +304,12 @@ mod tests { ) .unwrap(); assert_eq!(flags[0], "-lm"); - assert!(flags[1].ends_with("/tmp/project/libs/foo.a")); + assert_eq!( + flags[1], + absolutize_if_relative(project_dir, "libs/foo.a") + .to_string_lossy() + .to_string() + ); } #[test] @@ -326,9 +331,10 @@ mod tests { let overlay = scope_to_compile_overlay(project_dir, &scope).unwrap(); assert!(overlay.common.contains(&"-DFOO".to_string())); assert!(overlay.common.contains(&"-DBAR=1".to_string())); - assert!(overlay - .common - .contains(&"-I/tmp/project/include".to_string())); + assert!(overlay.common.contains(&format!( + "-I{}", + absolutize_if_relative(project_dir, "include").display() + ))); assert!(overlay.common.contains(&"-Wall".to_string())); assert_eq!(overlay.c, vec!["-std=c11"]); assert_eq!(overlay.cxx, vec!["-std=gnu++20"]); @@ -350,8 +356,16 @@ mod tests { let overlay = scope_to_link_overlay(project_dir, &scope).unwrap(); assert!(overlay.flags.contains(&"-Wl,--gc-sections".to_string())); - assert!(overlay.flags.contains(&"-L/tmp/project/lib".to_string())); + assert!(overlay.flags.contains(&format!( + "-L{}", + absolutize_if_relative(project_dir, "lib").display() + ))); assert_eq!(overlay.libs[0], "-lm"); - assert!(overlay.libs[1].ends_with("/tmp/project/archives/foo.a")); + assert_eq!( + overlay.libs[1], + absolutize_if_relative(project_dir, "archives/foo.a") + .to_string_lossy() + .to_string() + ); } }