From 2481099853bb747e0ccacc8e1e6a320d9d0b772e Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 17 Nov 2025 14:45:07 -0800 Subject: [PATCH 1/9] Proposal to standardize integrations --- Cargo.lock | 1 + Cargo.toml | 1 + FAQ_POC.md | 3 +- README.md | 10 +- crates/common/Cargo.toml | 1 + crates/common/src/error.rs | 8 + crates/common/src/html_processor.rs | 186 +++++++++++-- crates/common/src/integrations/mod.rs | 9 + crates/common/src/integrations/registry.rs | 157 +++++++++++ crates/common/src/integrations/starlight.rs | 293 ++++++++++++++++++++ crates/common/src/lib.rs | 1 + crates/common/src/publisher.rs | 15 +- crates/common/src/settings.rs | 13 +- crates/common/src/tsjs.rs | 52 +++- crates/fastly/src/main.rs | 50 ++-- crates/js/build.rs | 101 ++++--- crates/js/lib/package.json | 2 +- crates/js/lib/scripts/build-all.mjs | 45 +++ crates/js/lib/src/integrations/starlight.ts | 81 ++++++ crates/js/lib/vite.config.ts | 52 +++- crates/js/src/bundle.rs | 31 +-- crates/js/src/lib.rs | 3 +- crates/js/src/macros.rs | 68 ----- docs/integration_guide.md | 163 +++++++++++ fastly.toml | 3 +- trusted-server.toml | 5 + 26 files changed, 1139 insertions(+), 215 deletions(-) create mode 100644 crates/common/src/integrations/mod.rs create mode 100644 crates/common/src/integrations/registry.rs create mode 100644 crates/common/src/integrations/starlight.rs create mode 100644 crates/js/lib/scripts/build-all.mjs create mode 100644 crates/js/lib/src/integrations/starlight.ts delete mode 100644 crates/js/src/macros.rs create mode 100644 docs/integration_guide.md diff --git a/Cargo.lock b/Cargo.lock index 0844243..4ecf89e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2355,6 +2355,7 @@ checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" name = "trusted-server-common" version = "0.1.0" dependencies = [ + "async-trait", "base64 0.22.1", "brotli", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 44b723c..d2fa070 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ brotli = "8.0" bytes = "1.10" chacha20poly1305 = "0.10" chrono = "0.4.42" +async-trait = "0.1" config = "0.15.18" cookie = "0.18.1" derive_more = { version = "2.0", features = ["display", "error"] } diff --git a/FAQ_POC.md b/FAQ_POC.md index 39e4f2f..b79c1bf 100644 --- a/FAQ_POC.md +++ b/FAQ_POC.md @@ -25,7 +25,7 @@ NOT all the capabilities. Tech Lab will build and support core services to enabl **Does the Trusteed Server preclude using third party tags?** No, it does not. You should be able to begin migrating certain modules and parts of your content and experience as you go, without a forklift upgrade. We plan to support server side tagging capabilities to enable third party support. -**Why are you only using two vendors in the POC?** +**Why are you only using two partners in the POC?** Fastly and Equativ volunteered time and resources to us and they fit the technical needs and requirements for Trusted Server. For the sake of getting to market ASAP, we chose to double down on these two partners. We do not play favorites or have any financial incentive with these two companies and will begin implementing on other partners in the near future. Any ad exchange supporting prebid server requests should already find support. We will prioritize modules for other edge cloud providers based on industry priorities **How will this project be managed?** @@ -39,4 +39,3 @@ Yes. As long as your managed service provider can separate the edge from the CMS **How will this comply with Privacy regulations?** The trusted server will have modules to support Consent Management Providers (CMP) and send the GPP or TCF string as required in the ad request. - diff --git a/README.md b/README.md index d39abc1..d6be6d2 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ At this time, Trusted Server is designed to work with Fastly Compute. Follow the - Click Compute and Create Service - Click “Create Empty Service” (below main options) - Add your domain of the website you’ll be testing or using and click update - - Click on “Origins” section and add your ad-server / ssp partner information as hostnames (note after you save this information you can select port numbers and TLS on/off) - - IMPORTANT: when you enter the FQDN or IP ADDR information and click Add you need to enter a “Name” in the first field that will be referenced in your code so something like “my_ad_partner_1” + - Click on “Origins” section and add your ad-server / SSP integration information as hostnames (note after you save this information you can select port numbers and TLS on/off) + - IMPORTANT: when you enter the FQDN or IP ADDR information and click Add you need to enter a “Name” in the first field that will be referenced in your code so something like “my_ad_integration_1” - :warning: With a dev account, Fastly gives you a test domain by default, but you’re also able to create a CNAME to your own domain when you’re ready, along with 2 free TLS certs (non-wildcard). Note that Fastly Compute ONLY accepts client traffic via TLS, though origins and backends can be non-TLS. @@ -226,4 +226,8 @@ Once configured, the following endpoints are available: Notes - Rewriting uses `lol_html`. Only absolute and protocol‑relative URLs are rewritten; relative URLs are left unchanged. - For the proxy endpoint, the base URL is carried in `tsurl`, the original query parameters are preserved individually, and `tstoken` authenticates the reconstructed full URL. - - Synthetic identifiers are generated by `crates/common/src/synthetic.rs` and are surfaced in three places: publisher responses (headers + cookie), creative proxy target URLs (`synthetic_id` query param), and click redirect URLs. This ensures downstream partners can correlate impressions and clicks without direct third-party cookies. + - Synthetic identifiers are generated by `crates/common/src/synthetic.rs` and are surfaced in three places: publisher responses (headers + cookie), creative proxy target URLs (`synthetic_id` query param), and click redirect URLs. This ensures downstream integrations can correlate impressions and clicks without direct third-party cookies. + +## Integration Modules + +- See [`docs/integration_guide.md`](docs/integration_guide.md) for the full integration module guide, covering configuration, proxy routing, HTML shim hooks, and the `starlight` example implementation. diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index cdf21e3..6f40aef 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -14,6 +14,7 @@ brotli = { workspace = true } bytes = { workspace = true } chacha20poly1305 = { workspace = true } chrono = { workspace = true } +async-trait = { workspace = true } config = { workspace = true } cookie = { workspace = true } derive_more = { workspace = true } diff --git a/crates/common/src/error.rs b/crates/common/src/error.rs index 671f422..b57bc5d 100644 --- a/crates/common/src/error.rs +++ b/crates/common/src/error.rs @@ -50,6 +50,13 @@ pub enum TrustedServerError { #[display("Prebid error: {message}")] Prebid { message: String }, + /// Integration module error. + #[display("Integration error ({integration}): {message}")] + Integration { + integration: String, + message: String, + }, + /// Proxy error. #[display("Proxy error: {message}")] Proxy { message: String }, @@ -91,6 +98,7 @@ impl IntoHttpResponse for TrustedServerError { Self::InvalidUtf8 { .. } => StatusCode::BAD_REQUEST, Self::KvStore { .. } => StatusCode::SERVICE_UNAVAILABLE, Self::Prebid { .. } => StatusCode::BAD_GATEWAY, + Self::Integration { .. } => StatusCode::BAD_GATEWAY, Self::Proxy { .. } => StatusCode::BAD_GATEWAY, Self::SyntheticId { .. } => StatusCode::INTERNAL_SERVER_ERROR, Self::Template { .. } => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/crates/common/src/html_processor.rs b/crates/common/src/html_processor.rs index 5325a1e..bcf35a1 100644 --- a/crates/common/src/html_processor.rs +++ b/crates/common/src/html_processor.rs @@ -7,6 +7,7 @@ use std::rc::Rc; use lol_html::{element, html_content::ContentType, text, Settings as RewriterSettings}; use regex::Regex; +use crate::integrations::{IntegrationAttributeContext, IntegrationRegistry}; use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; @@ -18,6 +19,7 @@ pub struct HtmlProcessorConfig { pub request_host: String, pub request_scheme: String, pub enable_prebid: bool, + pub integrations: IntegrationRegistry, pub nextjs_enabled: bool, pub nextjs_attributes: Vec, } @@ -26,6 +28,7 @@ impl HtmlProcessorConfig { /// Create from settings and request parameters pub fn from_settings( settings: &Settings, + integrations: &IntegrationRegistry, origin_host: &str, request_host: &str, request_scheme: &str, @@ -35,6 +38,7 @@ impl HtmlProcessorConfig { request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), enable_prebid: settings.prebid.auto_configure, + integrations: integrations.clone(), nextjs_enabled: settings.publisher.nextjs.enabled, nextjs_attributes: settings.publisher.nextjs.rewrite_attributes.clone(), } @@ -114,6 +118,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let nextjs_attributes = Rc::new(config.nextjs_attributes.clone()); let injected_tsjs = Rc::new(Cell::new(false)); + let integration_registry = config.integrations.clone(); fn is_prebid_script_url(url: &str) -> bool { let lower = url.to_ascii_lowercase(); @@ -126,7 +131,6 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } let mut element_content_handlers = vec![ - // Inject tsjs once at the start of element!("head", { let injected_tsjs = injected_tsjs.clone(); move |el| { @@ -138,69 +142,121 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), - // Replace URLs in href attributes element!("[href]", { let patterns = patterns.clone(); let rewrite_prebid = config.enable_prebid; + let integrations = integration_registry.clone(); move |el| { - if let Some(href) = el.get_attribute("href") { - // If Prebid auto-config is enabled and this looks like a Prebid script href, rewrite to our extension + if let Some(mut href) = el.get_attribute("href") { + let original_href = href.clone(); if rewrite_prebid && is_prebid_script_url(&href) { - let ext_src = tsjs::ext_script_src(); - el.set_attribute("href", &ext_src)?; + href = tsjs::ext_script_src(); } else { let new_href = href .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()); if new_href != href { - el.set_attribute("href", &new_href)?; + href = new_href; } } + + if let Some(integration_href) = integrations.rewrite_attribute( + "href", + &href, + &IntegrationAttributeContext { + attribute_name: "href", + request_host: &patterns.request_host, + request_scheme: &patterns.request_scheme, + origin_host: &patterns.origin_host, + }, + ) { + href = integration_href; + } + + if href != original_href { + el.set_attribute("href", &href)?; + } } Ok(()) } }), - // Replace URLs in src attributes element!("[src]", { let patterns = patterns.clone(); let rewrite_prebid = config.enable_prebid; + let integrations = integration_registry.clone(); move |el| { - if let Some(src) = el.get_attribute("src") { + if let Some(mut src) = el.get_attribute("src") { + let original_src = src.clone(); if rewrite_prebid && is_prebid_script_url(&src) { - let ext_src = tsjs::ext_script_src(); - el.set_attribute("src", &ext_src)?; + src = tsjs::ext_script_src(); } else { let new_src = src .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()); if new_src != src { - el.set_attribute("src", &new_src)?; + src = new_src; } } + + if let Some(integration_src) = integrations.rewrite_attribute( + "src", + &src, + &IntegrationAttributeContext { + attribute_name: "src", + request_host: &patterns.request_host, + request_scheme: &patterns.request_scheme, + origin_host: &patterns.origin_host, + }, + ) { + src = integration_src; + } + + if src != original_src { + el.set_attribute("src", &src)?; + } } Ok(()) } }), - // Replace URLs in action attributes element!("[action]", { let patterns = patterns.clone(); + let integrations = integration_registry.clone(); move |el| { - if let Some(action) = el.get_attribute("action") { + if let Some(mut action) = el.get_attribute("action") { + let original_action = action.clone(); let new_action = action .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()); if new_action != action { - el.set_attribute("action", &new_action)?; + action = new_action; + } + + if let Some(integration_action) = integrations.rewrite_attribute( + "action", + &action, + &IntegrationAttributeContext { + attribute_name: "action", + request_host: &patterns.request_host, + request_scheme: &patterns.request_scheme, + origin_host: &patterns.origin_host, + }, + ) { + action = integration_action; + } + + if action != original_action { + el.set_attribute("action", &action)?; } } Ok(()) } }), - // Replace URLs in srcset attributes (for responsive images) element!("[srcset]", { let patterns = patterns.clone(); + let integrations = integration_registry.clone(); move |el| { - if let Some(srcset) = el.get_attribute("srcset") { + if let Some(mut srcset) = el.get_attribute("srcset") { + let original_srcset = srcset.clone(); let new_srcset = srcset .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()) @@ -209,19 +265,36 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &patterns.protocol_relative_replacement(), ) .replace(&patterns.origin_host, &patterns.request_host); - if new_srcset != srcset { - el.set_attribute("srcset", &new_srcset)?; + srcset = new_srcset; + } + + if let Some(integration_srcset) = integrations.rewrite_attribute( + "srcset", + &srcset, + &IntegrationAttributeContext { + attribute_name: "srcset", + request_host: &patterns.request_host, + request_scheme: &patterns.request_scheme, + origin_host: &patterns.origin_host, + }, + ) { + srcset = integration_srcset; + } + + if srcset != original_srcset { + el.set_attribute("srcset", &srcset)?; } } Ok(()) } }), - // Replace URLs in imagesrcset attributes (for link preload) element!("[imagesrcset]", { let patterns = patterns.clone(); + let integrations = integration_registry.clone(); move |el| { - if let Some(imagesrcset) = el.get_attribute("imagesrcset") { + if let Some(mut imagesrcset) = el.get_attribute("imagesrcset") { + let original_imagesrcset = imagesrcset.clone(); let new_imagesrcset = imagesrcset .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()) @@ -230,7 +303,24 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &patterns.protocol_relative_replacement(), ); if new_imagesrcset != imagesrcset { - el.set_attribute("imagesrcset", &new_imagesrcset)?; + imagesrcset = new_imagesrcset; + } + + if let Some(integration_imagesrcset) = integrations.rewrite_attribute( + "imagesrcset", + &imagesrcset, + &IntegrationAttributeContext { + attribute_name: "imagesrcset", + request_host: &patterns.request_host, + request_scheme: &patterns.request_scheme, + origin_host: &patterns.origin_host, + }, + ) { + imagesrcset = integration_imagesrcset; + } + + if imagesrcset != original_imagesrcset { + el.set_attribute("imagesrcset", &imagesrcset)?; } } Ok(()) @@ -301,6 +391,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso mod tests { use super::*; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::tsjs; use std::io::Cursor; fn create_test_config() -> HtmlProcessorConfig { @@ -309,6 +400,7 @@ mod tests { request_host: "test.example.com".to_string(), request_scheme: "https".to_string(), enable_prebid: false, + integrations: IntegrationRegistry::default(), nextjs_enabled: false, nextjs_attributes: vec!["href".to_string(), "link".to_string(), "url".to_string()], } @@ -558,8 +650,10 @@ mod tests { use crate::test_support::tests::create_test_settings; let settings = create_test_settings(); + let registry = IntegrationRegistry::new(&settings); let config = HtmlProcessorConfig::from_settings( &settings, + ®istry, "origin.test-publisher.com", "proxy.example.com", "https", @@ -652,6 +746,54 @@ mod tests { ); } + #[test] + fn test_integration_registry_rewrites_integration_scripts() { + use serde_json::json; + + let html = r#" + + "#; + + let mut settings = Settings::default(); + let shim_src = tsjs::script_src("tsjs-starlight.js") + .expect("Starlight tsjs bundle should exist for tests"); + settings.integrations.insert( + "starlight".to_string(), + json!({ + "endpoint": "https://example.com/openrtb2/auction", + "rewrite_scripts": true, + "shim_src": shim_src, + }), + ); + + let registry = IntegrationRegistry::new(&settings); + let mut config = create_test_config(); + config.integrations = registry; + + let processor = create_html_processor(config); + let pipeline_config = PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }; + let mut pipeline = StreamingPipeline::new(pipeline_config, processor); + + let mut output = Vec::new(); + let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); + assert!(result.is_ok()); + + let processed = String::from_utf8_lossy(&output); + let expected_src = tsjs::script_src("tsjs-starlight.js").expect("tsjs bundle missing"); + assert!( + processed.contains(&expected_src), + "Integration shim should replace integration script reference" + ); + assert!( + !processed.contains("cdn.starlight.com"), + "Original integration URL should be removed" + ); + } + #[test] fn test_real_publisher_html_with_gzip() { use flate2::read::GzDecoder; diff --git a/crates/common/src/integrations/mod.rs b/crates/common/src/integrations/mod.rs new file mode 100644 index 0000000..953c0bc --- /dev/null +++ b/crates/common/src/integrations/mod.rs @@ -0,0 +1,9 @@ +//! Integration module registry and sample implementations. + +mod registry; +pub mod starlight; + +pub use registry::{ + IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, + IntegrationProxy, IntegrationRegistry, +}; diff --git a/crates/common/src/integrations/registry.rs b/crates/common/src/integrations/registry.rs new file mode 100644 index 0000000..2e478b8 --- /dev/null +++ b/crates/common/src/integrations/registry.rs @@ -0,0 +1,157 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::Report; +use fastly::http::Method; +use fastly::{Request, Response}; + +use crate::error::TrustedServerError; +use crate::settings::Settings; + +/// Context provided to integration HTML attribute rewriters. +#[derive(Debug)] +pub struct IntegrationAttributeContext<'a> { + pub attribute_name: &'a str, + pub request_host: &'a str, + pub request_scheme: &'a str, + pub origin_host: &'a str, +} + +/// Describes an HTTP endpoint exposed by an integration. +#[derive(Clone)] +pub struct IntegrationEndpoint { + pub method: Method, + pub path: &'static str, +} + +impl IntegrationEndpoint { + #[must_use] + pub fn new(method: Method, path: &'static str) -> Self { + Self { method, path } + } + + #[must_use] + pub fn get(path: &'static str) -> Self { + Self { + method: Method::GET, + path, + } + } + + #[must_use] + pub fn post(path: &'static str) -> Self { + Self { + method: Method::POST, + path, + } + } +} + +/// Trait implemented by integration proxies that expose HTTP endpoints. +#[async_trait(?Send)] +pub trait IntegrationProxy: Send + Sync { + /// Routes handled by this integration (e.g. `/integrations/example/auction`). + fn routes(&self) -> Vec; + + /// Handle the proxied request. + async fn handle( + &self, + settings: &Settings, + req: Request, + ) -> Result>; +} + +/// Trait for integration-provided HTML attribute rewrite hooks. +pub trait IntegrationAttributeRewriter: Send + Sync { + /// Identifier for logging/diagnostics. + fn integration_id(&self) -> &'static str; + /// Return true when this rewriter wants to inspect a given attribute. + fn handles_attribute(&self, attribute: &str) -> bool; + /// Attempt to rewrite the attribute value. Return `Some(new_value)` to + /// update the attribute or `None` to keep the original value. + fn rewrite( + &self, + attr_name: &str, + attr_value: &str, + ctx: &IntegrationAttributeContext<'_>, + ) -> Option; +} + +struct RegisteredRoute { + method: Method, + path: &'static str, + proxy: Arc, +} + +#[derive(Default)] +struct IntegrationRegistryInner { + routes: Vec, + html_rewriters: Vec>, +} + + +/// In-memory registry of integrations discovered from settings. +#[derive(Clone, Default)] +pub struct IntegrationRegistry { + inner: Arc, +} + +impl IntegrationRegistry { + /// Build a registry from the provided settings. + pub fn new(settings: &Settings) -> Self { + let mut inner = IntegrationRegistryInner::default(); + + if let Some(integration) = crate::integrations::starlight::build(settings) { + for route in integration.routes() { + inner.routes.push(RegisteredRoute { + method: route.method.clone(), + path: route.path, + proxy: integration.clone(), + }); + } + inner.html_rewriters.push(integration); + } + + Self { + inner: Arc::new(inner), + } + } + + /// Return true when any proxy is registered for the provided route. + pub fn has_route(&self, method: &Method, path: &str) -> bool { + self.inner + .routes + .iter() + .any(|r| r.method == method && r.path == path) + } + + /// Dispatch a proxy request when an integration handles the path. + pub async fn handle_proxy( + &self, + method: &Method, + path: &str, + settings: &Settings, + req: Request, + ) -> Option>> { + for route in &self.inner.routes { + if route.method == method && route.path == path { + return Some(route.proxy.handle(settings, req).await); + } + } + None + } + + /// Give integrations a chance to rewrite HTML attributes. + pub fn rewrite_attribute( + &self, + attr_name: &str, + attr_value: &str, + ctx: &IntegrationAttributeContext<'_>, + ) -> Option { + self.inner + .html_rewriters + .iter() + .find(|rewriter| rewriter.handles_attribute(attr_name)) + .and_then(|rewriter| rewriter.rewrite(attr_name, attr_value, ctx)) + } +} diff --git a/crates/common/src/integrations/starlight.rs b/crates/common/src/integrations/starlight.rs new file mode 100644 index 0000000..9316704 --- /dev/null +++ b/crates/common/src/integrations/starlight.rs @@ -0,0 +1,293 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::{Report, ResultExt}; +use fastly::http::{header, Method}; +use fastly::{Request, Response}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use validator::Validate; + +use crate::backend::ensure_backend_from_url; +use crate::constants::{HEADER_SYNTHETIC_FRESH, HEADER_SYNTHETIC_TRUSTED_SERVER}; +use crate::error::TrustedServerError; +use crate::integrations::{ + IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, + IntegrationProxy, +}; +use crate::settings::Settings; +use crate::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; +use crate::tsjs; + +const STARLIGHT_INTEGRATION_ID: &str = "starlight"; + +#[derive(Debug, Deserialize)] +pub struct StarlightConfig { + pub endpoint: String, + #[serde(default = "default_timeout_ms")] + pub timeout_ms: u32, + #[serde(default = "default_shim_src")] + pub shim_src: String, + #[serde(default)] + pub rewrite_scripts: bool, +} + +#[derive(Debug, Deserialize, Serialize, Validate)] +struct StarlightRequestBody { + #[validate(nested)] + #[serde(default)] + user: StarlightUserSection, + #[validate(nested)] + #[serde(default)] + imp: Vec, + #[serde(flatten)] + extra: Map, +} + +#[derive(Debug, Default, Deserialize, Serialize, Validate)] +struct StarlightUserSection { + #[serde(default)] + #[validate(length(min = 1))] + id: Option, + #[serde(flatten)] + extra: Map, +} + +#[derive(Debug, Default, Deserialize, Serialize, Validate)] +struct StarlightImp { + #[serde(default)] + #[validate(length(min = 1))] + id: Option, + #[serde(flatten)] + extra: Map, +} + +#[derive(Debug, Deserialize, Serialize)] +struct StarlightResponseBody { + #[serde(flatten)] + fields: Map, +} + +pub struct StarlightIntegration { + config: StarlightConfig, +} + +impl StarlightIntegration { + fn new(config: StarlightConfig) -> Arc { + Arc::new(Self { config }) + } + + fn error(message: impl Into) -> TrustedServerError { + TrustedServerError::Integration { + integration: STARLIGHT_INTEGRATION_ID.to_string(), + message: message.into(), + } + } +} + +pub fn build(settings: &Settings) -> Option> { + let raw = settings.integration_config(STARLIGHT_INTEGRATION_ID)?; + let config: StarlightConfig = serde_json::from_value(raw.clone()).ok()?; + Some(StarlightIntegration::new(config)) +} + +#[async_trait(?Send)] +impl IntegrationProxy for StarlightIntegration { + fn routes(&self) -> Vec { + vec![IntegrationEndpoint::post("/integrations/starlight/auction")] + } + + async fn handle( + &self, + settings: &Settings, + mut req: Request, + ) -> Result> { + let mut payload = serde_json::from_slice::(&req.take_body_bytes()) + .change_context(Self::error("Failed to parse request body"))?; + payload + .validate() + .map_err(|err| Report::new(Self::error(format!("Invalid request payload: {err}"))))?; + + let synthetic_id = get_or_generate_synthetic_id(settings, &req) + .change_context(Self::error("Failed to fetch or mint synthetic ID"))?; + let fresh_id = generate_synthetic_id(settings, &req) + .change_context(Self::error("Failed to mint fresh synthetic ID"))?; + + payload.user.id = Some(synthetic_id.clone()); + + let mut upstream = Request::new(Method::POST, self.config.endpoint.clone()); + upstream.set_header(header::CONTENT_TYPE, "application/json"); + upstream + .set_body_json(&payload) + .change_context(Self::error("Failed to serialize request body"))?; + + if let Some(user_agent) = req.get_header(header::USER_AGENT) { + upstream.set_header(header::USER_AGENT, user_agent); + } + + let backend = ensure_backend_from_url(&self.config.endpoint) + .change_context(Self::error("Failed to determine backend"))?; + let mut response = upstream + .send(backend) + .change_context(Self::error("Failed to contact upstream integration"))?; + + // Attempt to parse response into structured form for logging/future transforms. + let response_body = response.take_body_bytes(); + match serde_json::from_slice::(&response_body) { + Ok(body) => { + response + .set_body_json(&body) + .change_context(Self::error("Failed to serialize integration response body"))?; + } + Err(_) => { + // Preserve original body if the integration responded with non-JSON content. + response.set_body(response_body); + } + } + + response.set_header(HEADER_SYNTHETIC_TRUSTED_SERVER, &synthetic_id); + response.set_header(HEADER_SYNTHETIC_FRESH, &fresh_id); + Ok(response) + } +} + +impl IntegrationAttributeRewriter for StarlightIntegration { + fn integration_id(&self) -> &'static str { + STARLIGHT_INTEGRATION_ID + } + + fn handles_attribute(&self, attribute: &str) -> bool { + self.config.rewrite_scripts && matches!(attribute, "src" | "href") + } + + fn rewrite( + &self, + _attr_name: &str, + attr_value: &str, + _ctx: &IntegrationAttributeContext<'_>, + ) -> Option { + if !self.config.rewrite_scripts { + return None; + } + + let lowered = attr_value.to_ascii_lowercase(); + if lowered.contains("starlight.js") { + Some(self.config.shim_src.clone()) + } else { + None + } + } +} + +fn default_timeout_ms() -> u32 { + 1000 +} + +fn default_shim_src() -> String { + tsjs::script_src("tsjs-starlight.js") + .unwrap_or_else(|| "/static/tsjs=tsjs-starlight.min.js".to_string()) +} + +impl Default for StarlightRequestBody { + fn default() -> Self { + Self { + user: StarlightUserSection::default(), + imp: Vec::new(), + extra: Map::new(), + } + } +} + +impl Default for StarlightResponseBody { + fn default() -> Self { + Self { fields: Map::new() } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{test_support::tests::create_test_settings, tsjs}; + use fastly::http::Method; + use serde_json::json; + + #[test] + fn build_requires_config() { + let settings = create_test_settings(); + assert!( + build(&settings).is_none(), + "Should not build without integration config" + ); + } + + #[test] + fn html_rewriter_replaces_integration_script() { + let shim_src = tsjs::script_src("tsjs-starlight.js") + .expect("tsjs starlight bundle should exist for tests"); + let config = StarlightConfig { + endpoint: "https://example.com/openrtb".to_string(), + timeout_ms: 1000, + shim_src: shim_src.clone(), + rewrite_scripts: true, + }; + let integration = StarlightIntegration::new(config); + + let ctx = IntegrationAttributeContext { + attribute_name: "src", + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + }; + + let rewritten = + integration.rewrite("src", "https://cdn.starlight.net/v1/starlight.js", &ctx); + assert_eq!( + rewritten.as_deref(), + Some(shim_src.as_str()), + "Should swap integration script for trusted shim" + ); + } + + #[test] + fn html_rewriter_is_noop_when_disabled() { + let shim_src = tsjs::script_src("tsjs-starlight.js") + .expect("tsjs starlight bundle should exist for tests"); + let config = StarlightConfig { + endpoint: "https://example.com/openrtb".to_string(), + timeout_ms: 1000, + shim_src, + rewrite_scripts: false, + }; + let integration = StarlightIntegration::new(config); + let ctx = IntegrationAttributeContext { + attribute_name: "src", + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + }; + + assert!(integration + .rewrite("src", "https://cdn.starlight.net/script.js", &ctx) + .is_none()); + } + + #[test] + fn build_uses_settings_integration_block() { + let mut settings = create_test_settings(); + settings.integrations.insert( + STARLIGHT_INTEGRATION_ID.to_string(), + json!({ + "endpoint": "https://example.com/bid", + "rewrite_scripts": true, + }), + ); + + let integration = build(&settings).expect("Integration should build with config"); + let routes = integration.routes(); + assert!( + routes.iter().any(|route| route.method == Method::POST + && route.path == "/integrations/starlight/auction"), + "Integration should register POST /integrations/starlight/auction" + ); + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 124b839..3af42b8 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -34,6 +34,7 @@ pub mod fastly_storage; pub mod geo; pub mod html_processor; pub mod http_util; +pub mod integrations; pub mod models; pub mod openrtb; pub mod prebid_proxy; diff --git a/crates/common/src/publisher.rs b/crates/common/src/publisher.rs index 2820d87..a8c13a2 100644 --- a/crates/common/src/publisher.rs +++ b/crates/common/src/publisher.rs @@ -8,6 +8,7 @@ use crate::http_util::serve_static_with_etag; use crate::constants::{HEADER_SYNTHETIC_TRUSTED_SERVER, HEADER_X_COMPRESS_HINT}; use crate::cookies::create_synthetic_cookie; use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistry; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; use crate::streaming_replacer::create_url_replacer; @@ -105,6 +106,7 @@ struct ProcessResponseParams<'a> { request_scheme: &'a str, settings: &'a Settings, content_type: &'a str, + integration_registry: &'a IntegrationRegistry, } /// Process response body in streaming fashion with compression preservation @@ -129,6 +131,7 @@ fn process_response_streaming( params.request_host, params.request_scheme, params.settings, + params.integration_registry, )?; let config = PipelineConfig { @@ -171,11 +174,17 @@ fn create_html_stream_processor( request_host: &str, request_scheme: &str, settings: &Settings, + integration_registry: &IntegrationRegistry, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; - let config = - HtmlProcessorConfig::from_settings(settings, origin_host, request_host, request_scheme); + let config = HtmlProcessorConfig::from_settings( + settings, + integration_registry, + origin_host, + request_host, + request_scheme, + ); Ok(create_html_processor(config)) } @@ -193,6 +202,7 @@ fn create_html_stream_processor( /// - The origin backend is unreachable pub fn handle_publisher_request( settings: &Settings, + integration_registry: &IntegrationRegistry, mut req: Request, ) -> Result> { log::info!("Proxying request to publisher_origin"); @@ -300,6 +310,7 @@ pub fn handle_publisher_request( request_scheme: &request_scheme, settings, content_type: &content_type, + integration_registry, }; match process_response_streaming(body, params) { Ok(processed_body) => { diff --git a/crates/common/src/settings.rs b/crates/common/src/settings.rs index a9c8964..3479cb8 100644 --- a/crates/common/src/settings.rs +++ b/crates/common/src/settings.rs @@ -188,6 +188,8 @@ pub struct Settings { #[serde(default)] #[validate(nested)] pub synthetic: Synthetic, + #[serde(default)] + pub integrations: HashMap, #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub handlers: Vec, @@ -260,6 +262,11 @@ impl Settings { .iter() .find(|handler| handler.matches_path(path)) } + + #[must_use] + pub fn integration_config(&self, integration_id: &str) -> Option<&JsonValue> { + self.integrations.get(integration_id) + } } fn validate_path(value: &str) -> Result<(), ValidationError> { @@ -587,10 +594,6 @@ mod tests { #[test] fn test_set_env() { - let re = Regex::new(r"ad_partner_url = .*").unwrap(); - let toml_str = crate_test_settings_str(); - let toml_str = re.replace(&toml_str, ""); - temp_env::with_var( format!( "{}{}PUBLISHER{}ORIGIN_URL", @@ -600,7 +603,7 @@ mod tests { ), Some("https://change-publisher.com"), || { - let settings = Settings::from_toml(&toml_str); + let settings = Settings::from_toml(&crate_test_settings_str()); assert!(settings.is_ok(), "Settings should load from embedded TOML"); assert_eq!( diff --git a/crates/common/src/tsjs.rs b/crates/common/src/tsjs.rs index c4c4cef..336e4ab 100644 --- a/crates/common/src/tsjs.rs +++ b/crates/common/src/tsjs.rs @@ -1,14 +1,15 @@ -use trusted_server_js::{bundle_hash, TsjsBundle}; +use trusted_server_js::{bundle_for_filename, bundle_hash}; -fn script_src_for(bundle: TsjsBundle) -> String { - format!( - "/static/tsjs={}?v={}", - bundle.minified_filename(), - bundle_hash(bundle) - ) +fn script_src_internal(filename: &str) -> String { + script_src(filename).unwrap_or_else(|| { + panic!( + "tsjs: bundle {} not found. Ensure npm build produced this bundle.", + filename + ) + }) } -fn script_tag_for(bundle: TsjsBundle, attrs: &str) -> String { +fn script_tag_internal(filename: &str, attrs: &str) -> String { let attr_segment = if attrs.is_empty() { String::new() } else { @@ -16,32 +17,55 @@ fn script_tag_for(bundle: TsjsBundle, attrs: &str) -> String { }; format!( "", - script_src_for(bundle), + script_src_internal(filename), attr_segment ) } +/// Returns `/static/tsjs=?v=` if the bundle is available. +pub fn script_src(filename: &str) -> Option { + let trimmed = filename.trim(); + bundle_for_filename(trimmed)?; + let hash = bundle_hash(trimmed)?; + let minified = trimmed + .strip_suffix(".js") + .map(|stem| format!("{stem}.min.js")) + .unwrap_or_else(|| format!("{trimmed}.min.js")); + Some(format!("/static/tsjs={}?v={}", minified, hash)) +} + +/// Returns a `", src) + } else { + format!("", src, attrs) + } + }) +} + /// ` + "#; + let _bundle_guard = mock_testlight_bundle(); let mut settings = Settings::default(); - let shim_src = tsjs::script_src("tsjs-starlight.js") - .expect("Starlight tsjs bundle should exist for tests"); - settings.integrations.insert( - "starlight".to_string(), - json!({ - "endpoint": "https://example.com/openrtb2/auction", - "rewrite_scripts": true, - "shim_src": shim_src, - }), - ); + let shim_src = tsjs::integration_script_src("testlight"); + settings + .integrations + .insert_config( + "testlight", + &json!({ + "enabled": true, + "endpoint": "https://example.com/openrtb2/auction", + "rewrite_scripts": true, + "shim_src": shim_src, + }), + ) + .expect("should insert testlight config"); let registry = IntegrationRegistry::new(&settings); let mut config = create_test_config(); @@ -783,13 +849,13 @@ mod tests { assert!(result.is_ok()); let processed = String::from_utf8_lossy(&output); - let expected_src = tsjs::script_src("tsjs-starlight.js").expect("tsjs bundle missing"); + let expected_src = tsjs::integration_script_src("testlight"); assert!( processed.contains(&expected_src), "Integration shim should replace integration script reference" ); assert!( - !processed.contains("cdn.starlight.com"), + !processed.contains("cdn.testlight.com"), "Original integration URL should be removed" ); } diff --git a/crates/common/src/integrations/mod.rs b/crates/common/src/integrations/mod.rs index 953c0bc..190e160 100644 --- a/crates/common/src/integrations/mod.rs +++ b/crates/common/src/integrations/mod.rs @@ -1,9 +1,18 @@ //! Integration module registry and sample implementations. +use crate::settings::Settings; + mod registry; -pub mod starlight; +pub mod testlight; pub use registry::{ IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, - IntegrationProxy, IntegrationRegistry, + IntegrationMetadata, IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, + IntegrationRegistry, IntegrationScriptContext, IntegrationScriptRewriter, }; + +type IntegrationBuilder = fn(&Settings) -> Option; + +pub(crate) fn builders() -> &'static [IntegrationBuilder] { + &[testlight::register] +} diff --git a/crates/common/src/integrations/registry.rs b/crates/common/src/integrations/registry.rs index 32178ff..16e2ca5 100644 --- a/crates/common/src/integrations/registry.rs +++ b/crates/common/src/integrations/registry.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::sync::Arc; use async_trait::async_trait; @@ -17,8 +18,17 @@ pub struct IntegrationAttributeContext<'a> { pub origin_host: &'a str, } +/// Context passed to script/text rewriters for inline HTML handling. +#[derive(Debug)] +pub struct IntegrationScriptContext<'a> { + pub selector: &'a str, + pub request_host: &'a str, + pub request_scheme: &'a str, + pub origin_host: &'a str, +} + /// Describes an HTTP endpoint exposed by an integration. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct IntegrationEndpoint { pub method: Method, pub path: &'static str, @@ -77,16 +87,117 @@ pub trait IntegrationAttributeRewriter: Send + Sync { ) -> Option; } +/// Trait for integration-provided inline script/text rewrite hooks. +pub trait IntegrationScriptRewriter: Send + Sync { + /// Identifier for logging/diagnostics. + fn integration_id(&self) -> &'static str; + /// CSS selector (e.g. `script#__NEXT_DATA__`) that should trigger this rewriter. + fn selector(&self) -> &'static str; + /// Attempt to rewrite the inline text content for the selector. + fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> Option; +} + +/// Registration payload returned by integration builders. +pub struct IntegrationRegistration { + pub integration_id: &'static str, + pub proxies: Vec>, + pub attribute_rewriters: Vec>, + pub script_rewriters: Vec>, + pub assets: Vec, +} + +impl IntegrationRegistration { + #[must_use] + pub fn builder(integration_id: &'static str) -> IntegrationRegistrationBuilder { + IntegrationRegistrationBuilder::new(integration_id) + } +} + +pub struct IntegrationRegistrationBuilder { + registration: IntegrationRegistration, +} + +impl IntegrationRegistrationBuilder { + fn new(integration_id: &'static str) -> Self { + Self { + registration: IntegrationRegistration { + integration_id, + proxies: Vec::new(), + attribute_rewriters: Vec::new(), + script_rewriters: Vec::new(), + assets: Vec::new(), + }, + } + } + + #[must_use] + pub fn with_proxy(mut self, proxy: Arc) -> Self { + self.registration.proxies.push(proxy); + self + } + + #[must_use] + pub fn with_attribute_rewriter( + mut self, + rewriter: Arc, + ) -> Self { + self.registration.attribute_rewriters.push(rewriter); + self + } + + #[must_use] + pub fn with_script_rewriter(mut self, rewriter: Arc) -> Self { + self.registration.script_rewriters.push(rewriter); + self + } + + #[must_use] + pub fn with_asset(mut self, asset: impl Into) -> Self { + self.registration.assets.push(asset.into()); + self + } + + #[must_use] + pub fn build(self) -> IntegrationRegistration { + self.registration + } +} + struct RegisteredRoute { method: Method, path: &'static str, proxy: Arc, + integration_id: &'static str, } #[derive(Default)] struct IntegrationRegistryInner { routes: Vec, html_rewriters: Vec>, + script_rewriters: Vec>, + assets: Vec<(&'static str, String)>, +} + +/// Summary of registered integration capabilities. +#[derive(Debug, Clone)] +pub struct IntegrationMetadata { + pub id: &'static str, + pub routes: Vec, + pub attribute_rewriters: usize, + pub script_selectors: Vec<&'static str>, + pub assets: Vec, +} + +impl IntegrationMetadata { + fn new(id: &'static str) -> Self { + Self { + id, + routes: Vec::new(), + attribute_rewriters: 0, + script_selectors: Vec::new(), + assets: Vec::new(), + } + } } /// In-memory registry of integrations discovered from settings. @@ -100,15 +211,31 @@ impl IntegrationRegistry { pub fn new(settings: &Settings) -> Self { let mut inner = IntegrationRegistryInner::default(); - if let Some(integration) = crate::integrations::starlight::build(settings) { - for route in integration.routes() { - inner.routes.push(RegisteredRoute { - method: route.method.clone(), - path: route.path, - proxy: integration.clone(), - }); + for builder in crate::integrations::builders() { + if let Some(registration) = builder(settings) { + for proxy in registration.proxies { + for route in proxy.routes() { + inner.routes.push(RegisteredRoute { + method: route.method.clone(), + path: route.path, + proxy: proxy.clone(), + integration_id: registration.integration_id, + }); + } + } + inner + .html_rewriters + .extend(registration.attribute_rewriters.into_iter()); + inner + .script_rewriters + .extend(registration.script_rewriters.into_iter()); + inner.assets.extend( + registration + .assets + .into_iter() + .map(|asset| (registration.integration_id, asset)), + ); } - inner.html_rewriters.push(integration); } Self { @@ -147,10 +274,64 @@ impl IntegrationRegistry { attr_value: &str, ctx: &IntegrationAttributeContext<'_>, ) -> Option { - self.inner - .html_rewriters - .iter() - .find(|rewriter| rewriter.handles_attribute(attr_name)) - .and_then(|rewriter| rewriter.rewrite(attr_name, attr_value, ctx)) + let mut current = attr_value.to_string(); + let mut changed = false; + for rewriter in &self.inner.html_rewriters { + if !rewriter.handles_attribute(attr_name) { + continue; + } + if let Some(next_value) = rewriter.rewrite(attr_name, ¤t, ctx) { + current = next_value; + changed = true; + } + } + + if changed { + Some(current) + } else { + None + } + } + + /// Expose registered script/text rewriters for HTML processing. + pub fn script_rewriters(&self) -> Vec> { + self.inner.script_rewriters.clone() + } + + /// Provide a snapshot of registered integrations and their hooks. + pub fn registered_integrations(&self) -> Vec { + let mut map: BTreeMap<&'static str, IntegrationMetadata> = BTreeMap::new(); + + for route in &self.inner.routes { + let entry = map + .entry(route.integration_id) + .or_insert_with(|| IntegrationMetadata::new(route.integration_id)); + entry + .routes + .push(IntegrationEndpoint::new(route.method.clone(), route.path)); + } + + for rewriter in &self.inner.html_rewriters { + let entry = map + .entry(rewriter.integration_id()) + .or_insert_with(|| IntegrationMetadata::new(rewriter.integration_id())); + entry.attribute_rewriters += 1; + } + + for rewriter in &self.inner.script_rewriters { + let entry = map + .entry(rewriter.integration_id()) + .or_insert_with(|| IntegrationMetadata::new(rewriter.integration_id())); + entry.script_selectors.push(rewriter.selector()); + } + + for (integration_id, asset) in &self.inner.assets { + let entry = map + .entry(*integration_id) + .or_insert_with(|| IntegrationMetadata::new(*integration_id)); + entry.assets.push(asset.clone()); + } + + map.into_values().collect() } } diff --git a/crates/common/src/integrations/starlight.rs b/crates/common/src/integrations/testlight.rs similarity index 64% rename from crates/common/src/integrations/starlight.rs rename to crates/common/src/integrations/testlight.rs index 9316704..a216fd4 100644 --- a/crates/common/src/integrations/starlight.rs +++ b/crates/common/src/integrations/testlight.rs @@ -13,39 +13,50 @@ use crate::constants::{HEADER_SYNTHETIC_FRESH, HEADER_SYNTHETIC_TRUSTED_SERVER}; use crate::error::TrustedServerError; use crate::integrations::{ IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, - IntegrationProxy, + IntegrationProxy, IntegrationRegistration, }; -use crate::settings::Settings; +use crate::settings::{IntegrationConfig as IntegrationConfigTrait, Settings}; use crate::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; use crate::tsjs; -const STARLIGHT_INTEGRATION_ID: &str = "starlight"; +const TESTLIGHT_INTEGRATION_ID: &str = "testlight"; -#[derive(Debug, Deserialize)] -pub struct StarlightConfig { +#[derive(Debug, Deserialize, Validate)] +pub struct TestlightConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, + #[validate(url)] pub endpoint: String, #[serde(default = "default_timeout_ms")] + #[validate(range(min = 10, max = 60000))] pub timeout_ms: u32, #[serde(default = "default_shim_src")] + #[validate(length(min = 1))] pub shim_src: String, #[serde(default)] pub rewrite_scripts: bool, } +impl IntegrationConfigTrait for TestlightConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + #[derive(Debug, Deserialize, Serialize, Validate)] -struct StarlightRequestBody { +struct TestlightRequestBody { #[validate(nested)] #[serde(default)] - user: StarlightUserSection, + user: TestlightUserSection, #[validate(nested)] #[serde(default)] - imp: Vec, + imp: Vec, #[serde(flatten)] extra: Map, } #[derive(Debug, Default, Deserialize, Serialize, Validate)] -struct StarlightUserSection { +struct TestlightUserSection { #[serde(default)] #[validate(length(min = 1))] id: Option, @@ -54,7 +65,7 @@ struct StarlightUserSection { } #[derive(Debug, Default, Deserialize, Serialize, Validate)] -struct StarlightImp { +struct TestlightImp { #[serde(default)] #[validate(length(min = 1))] id: Option, @@ -63,38 +74,56 @@ struct StarlightImp { } #[derive(Debug, Deserialize, Serialize)] -struct StarlightResponseBody { +struct TestlightResponseBody { #[serde(flatten)] fields: Map, } -pub struct StarlightIntegration { - config: StarlightConfig, +pub struct TestlightIntegration { + config: TestlightConfig, } -impl StarlightIntegration { - fn new(config: StarlightConfig) -> Arc { +impl TestlightIntegration { + fn new(config: TestlightConfig) -> Arc { Arc::new(Self { config }) } fn error(message: impl Into) -> TrustedServerError { TrustedServerError::Integration { - integration: STARLIGHT_INTEGRATION_ID.to_string(), + integration: TESTLIGHT_INTEGRATION_ID.to_string(), message: message.into(), } } } -pub fn build(settings: &Settings) -> Option> { - let raw = settings.integration_config(STARLIGHT_INTEGRATION_ID)?; - let config: StarlightConfig = serde_json::from_value(raw.clone()).ok()?; - Some(StarlightIntegration::new(config)) +fn build(settings: &Settings) -> Option> { + let config = match settings.integration_config::(TESTLIGHT_INTEGRATION_ID) { + Ok(Some(config)) => config, + Ok(None) => return None, + Err(err) => { + log::error!("Failed to load Testlight integration config: {err:?}"); + return None; + } + }; + + Some(TestlightIntegration::new(config)) +} + +pub fn register(settings: &Settings) -> Option { + let integration = build(settings)?; + Some( + IntegrationRegistration::builder(TESTLIGHT_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration) + .with_asset("testlight") + .build(), + ) } #[async_trait(?Send)] -impl IntegrationProxy for StarlightIntegration { +impl IntegrationProxy for TestlightIntegration { fn routes(&self) -> Vec { - vec![IntegrationEndpoint::post("/integrations/starlight/auction")] + vec![IntegrationEndpoint::post("/integrations/testlight/auction")] } async fn handle( @@ -102,7 +131,7 @@ impl IntegrationProxy for StarlightIntegration { settings: &Settings, mut req: Request, ) -> Result> { - let mut payload = serde_json::from_slice::(&req.take_body_bytes()) + let mut payload = serde_json::from_slice::(&req.take_body_bytes()) .change_context(Self::error("Failed to parse request body"))?; payload .validate() @@ -133,7 +162,7 @@ impl IntegrationProxy for StarlightIntegration { // Attempt to parse response into structured form for logging/future transforms. let response_body = response.take_body_bytes(); - match serde_json::from_slice::(&response_body) { + match serde_json::from_slice::(&response_body) { Ok(body) => { response .set_body_json(&body) @@ -151,9 +180,9 @@ impl IntegrationProxy for StarlightIntegration { } } -impl IntegrationAttributeRewriter for StarlightIntegration { +impl IntegrationAttributeRewriter for TestlightIntegration { fn integration_id(&self) -> &'static str { - STARLIGHT_INTEGRATION_ID + TESTLIGHT_INTEGRATION_ID } fn handles_attribute(&self, attribute: &str) -> bool { @@ -171,7 +200,7 @@ impl IntegrationAttributeRewriter for StarlightIntegration { } let lowered = attr_value.to_ascii_lowercase(); - if lowered.contains("starlight.js") { + if lowered.contains("testlight.js") { Some(self.config.shim_src.clone()) } else { None @@ -184,21 +213,24 @@ fn default_timeout_ms() -> u32 { } fn default_shim_src() -> String { - tsjs::script_src("tsjs-starlight.js") - .unwrap_or_else(|| "/static/tsjs=tsjs-starlight.min.js".to_string()) + tsjs::integration_script_src("testlight") } -impl Default for StarlightRequestBody { +fn default_enabled() -> bool { + true +} + +impl Default for TestlightRequestBody { fn default() -> Self { Self { - user: StarlightUserSection::default(), + user: TestlightUserSection::default(), imp: Vec::new(), extra: Map::new(), } } } -impl Default for StarlightResponseBody { +impl Default for TestlightResponseBody { fn default() -> Self { Self { fields: Map::new() } } @@ -211,6 +243,21 @@ mod tests { use fastly::http::Method; use serde_json::json; + const MOCK_TESTLIGHT_SRC: &str = "https://mock.testassets/testlight.js"; + + struct MockBundleGuard; + + fn mock_testlight_bundle() -> MockBundleGuard { + tsjs::mock_integration_bundle("testlight", MOCK_TESTLIGHT_SRC); + MockBundleGuard + } + + impl Drop for MockBundleGuard { + fn drop(&mut self) { + tsjs::clear_mock_integration_bundles(); + } + } + #[test] fn build_requires_config() { let settings = create_test_settings(); @@ -222,15 +269,16 @@ mod tests { #[test] fn html_rewriter_replaces_integration_script() { - let shim_src = tsjs::script_src("tsjs-starlight.js") - .expect("tsjs starlight bundle should exist for tests"); - let config = StarlightConfig { + let _bundle_guard = mock_testlight_bundle(); + let shim_src = tsjs::integration_script_src("testlight"); + let config = TestlightConfig { + enabled: true, endpoint: "https://example.com/openrtb".to_string(), timeout_ms: 1000, shim_src: shim_src.clone(), rewrite_scripts: true, }; - let integration = StarlightIntegration::new(config); + let integration = TestlightIntegration::new(config); let ctx = IntegrationAttributeContext { attribute_name: "src", @@ -240,7 +288,7 @@ mod tests { }; let rewritten = - integration.rewrite("src", "https://cdn.starlight.net/v1/starlight.js", &ctx); + integration.rewrite("src", "https://cdn.testlight.net/v1/testlight.js", &ctx); assert_eq!( rewritten.as_deref(), Some(shim_src.as_str()), @@ -250,15 +298,16 @@ mod tests { #[test] fn html_rewriter_is_noop_when_disabled() { - let shim_src = tsjs::script_src("tsjs-starlight.js") - .expect("tsjs starlight bundle should exist for tests"); - let config = StarlightConfig { + let _bundle_guard = mock_testlight_bundle(); + let shim_src = tsjs::integration_script_src("testlight"); + let config = TestlightConfig { + enabled: true, endpoint: "https://example.com/openrtb".to_string(), timeout_ms: 1000, shim_src, rewrite_scripts: false, }; - let integration = StarlightIntegration::new(config); + let integration = TestlightIntegration::new(config); let ctx = IntegrationAttributeContext { attribute_name: "src", request_host: "edge.example.com", @@ -267,27 +316,31 @@ mod tests { }; assert!(integration - .rewrite("src", "https://cdn.starlight.net/script.js", &ctx) + .rewrite("src", "https://cdn.testlight.net/script.js", &ctx) .is_none()); } #[test] fn build_uses_settings_integration_block() { let mut settings = create_test_settings(); - settings.integrations.insert( - STARLIGHT_INTEGRATION_ID.to_string(), - json!({ - "endpoint": "https://example.com/bid", - "rewrite_scripts": true, - }), - ); + settings + .integrations + .insert_config( + TESTLIGHT_INTEGRATION_ID.to_string(), + &json!({ + "enabled": true, + "endpoint": "https://example.com/bid", + "rewrite_scripts": true, + }), + ) + .expect("should insert integration config"); let integration = build(&settings).expect("Integration should build with config"); let routes = integration.routes(); assert!( routes.iter().any(|route| route.method == Method::POST - && route.path == "/integrations/starlight/auction"), - "Integration should register POST /integrations/starlight/auction" + && route.path == "/integrations/testlight/auction"), + "Integration should register POST /integrations/testlight/auction" ); } } diff --git a/crates/common/src/settings.rs b/crates/common/src/settings.rs index 3479cb8..fdf8a88 100644 --- a/crates/common/src/settings.rs +++ b/crates/common/src/settings.rs @@ -9,6 +9,7 @@ use serde::{ }; use serde_json::Value as JsonValue; use std::collections::HashMap; +use std::ops::{Deref, DerefMut}; use std::sync::OnceLock; use url::Url; use validator::{Validate, ValidationError}; @@ -110,6 +111,107 @@ impl Default for NextJs { } } +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct IntegrationSettings { + #[serde(flatten)] + entries: HashMap, +} + +pub trait IntegrationConfig: DeserializeOwned + Validate { + fn is_enabled(&self) -> bool; +} + +impl IntegrationSettings { + #[cfg_attr(not(test), allow(dead_code))] + pub fn insert_config( + &mut self, + integration_id: impl Into, + value: &T, + ) -> Result<(), Report> + where + T: Serialize, + { + let json = + serde_json::to_value(value).change_context(TrustedServerError::Configuration { + message: "Failed to serialize integration configuration".to_string(), + })?; + self.entries.insert(integration_id.into(), json); + Ok(()) + } + + fn normalize_env_value(value: JsonValue) -> JsonValue { + match value { + JsonValue::Object(map) => JsonValue::Object( + map.into_iter() + .map(|(key, val)| (key, Self::normalize_env_value(val))) + .collect(), + ), + JsonValue::Array(items) => { + JsonValue::Array(items.into_iter().map(Self::normalize_env_value).collect()) + } + JsonValue::String(raw) => { + if let Ok(parsed) = serde_json::from_str::(&raw) { + parsed + } else { + JsonValue::String(raw) + } + } + other => other, + } + } + + pub fn get_typed( + &self, + integration_id: &str, + ) -> Result, Report> + where + T: IntegrationConfig, + { + let raw = match self.entries.get(integration_id) { + Some(value) => value, + None => return Ok(None), + }; + + let normalized = Self::normalize_env_value(raw.clone()); + + let config: T = serde_json::from_value(normalized).change_context( + TrustedServerError::Configuration { + message: format!( + "Integration '{integration_id}' configuration could not be parsed" + ), + }, + )?; + + config.validate().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!( + "Integration '{integration_id}' configuration failed validation: {err}" + ), + }) + })?; + + if !config.is_enabled() { + return Ok(None); + } + + Ok(Some(config)) + } +} + +impl Deref for IntegrationSettings { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.entries + } +} + +impl DerefMut for IntegrationSettings { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.entries + } +} + fn deserialize_nextjs_attributes<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -189,7 +291,7 @@ pub struct Settings { #[validate(nested)] pub synthetic: Synthetic, #[serde(default)] - pub integrations: HashMap, + pub integrations: IntegrationSettings, #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub handlers: Vec, @@ -263,9 +365,14 @@ impl Settings { .find(|handler| handler.matches_path(path)) } - #[must_use] - pub fn integration_config(&self, integration_id: &str) -> Option<&JsonValue> { - self.integrations.get(integration_id) + pub fn integration_config( + &self, + integration_id: &str, + ) -> Result, Report> + where + T: IntegrationConfig, + { + self.integrations.get_typed(integration_id) } } @@ -340,7 +447,7 @@ mod tests { use super::*; use regex::Regex; - use crate::test_support::tests::crate_test_settings_str; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; #[test] fn test_settings_new() { @@ -700,4 +807,90 @@ mod tests { }; assert_eq!(publisher.origin_host(), "[::1]:8080"); } + + #[test] + fn test_integration_settings_from_env() { + use crate::integrations::testlight::TestlightConfig; + + let toml_str = crate_test_settings_str(); + + let origin_key = format!( + "{}{}PUBLISHER{}ORIGIN_URL", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + let integration_prefix = format!( + "{}{}INTEGRATIONS{}TESTLIGHT{}", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + let endpoint_key = format!("{}ENDPOINT", integration_prefix); + let timeout_key = format!("{}TIMEOUT_MS", integration_prefix); + let rewrite_key = format!("{}REWRITE_SCRIPTS", integration_prefix); + let enabled_key = format!("{}ENABLED", integration_prefix); + + temp_env::with_var( + origin_key, + Some("https://origin.test-publisher.com"), + || { + temp_env::with_var( + endpoint_key, + Some("https://testlight-env.test/auction"), + || { + temp_env::with_var(timeout_key, Some("2500"), || { + temp_env::with_var(rewrite_key, Some("true"), || { + temp_env::with_var(enabled_key, Some("true"), || { + let settings = Settings::from_toml(&toml_str) + .expect("Settings should load"); + + let config = settings + .integration_config::("testlight") + .expect("integration parsing should succeed") + .expect("integration should be enabled"); + + assert_eq!( + config.endpoint, + "https://testlight-env.test/auction" + ); + assert_eq!(config.timeout_ms, 2500); + assert!(config.rewrite_scripts); + assert!(config.enabled); + }); + }); + }); + }, + ); + }, + ); + } + + #[test] + fn test_disabled_integration_does_not_register() { + use crate::integrations::testlight::TestlightConfig; + use serde_json::json; + + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "testlight", + &json!({ + "enabled": false, + "endpoint": "https://testlight.test/auction", + "rewrite_scripts": true, + }), + ) + .expect("should insert integration config"); + + let config = settings + .integration_config::("testlight") + .expect("integration parsing should succeed"); + + assert!(config.is_none(), "Disabled integrations should be skipped"); + } } diff --git a/crates/common/src/tsjs.rs b/crates/common/src/tsjs.rs index 336e4ab..ffb1e68 100644 --- a/crates/common/src/tsjs.rs +++ b/crates/common/src/tsjs.rs @@ -1,5 +1,16 @@ use trusted_server_js::{bundle_for_filename, bundle_hash}; +#[cfg(test)] +use once_cell::sync::Lazy; +#[cfg(test)] +use std::collections::HashMap; +#[cfg(test)] +use std::sync::RwLock; + +#[cfg(test)] +static MOCK_INTEGRATION_BUNDLES: Lazy>> = + Lazy::new(|| RwLock::new(HashMap::new())); + fn script_src_internal(filename: &str) -> String { script_src(filename).unwrap_or_else(|| { panic!( @@ -69,3 +80,55 @@ pub fn ext_script_src() -> String { pub fn creative_script_src() -> String { script_src_internal("tsjs-creative.js") } + +fn integration_bundle_filename(name: &str) -> String { + format!("tsjs-{name}.js") +} + +fn integration_fallback_src(name: &str) -> String { + format!("/static/tsjs=tsjs-{name}.min.js") +} + +/// Returns the script URL for an integration bundle, falling back to the +/// static path when the hashed bundle isn't available (e.g., in tests). +pub fn integration_script_src(name: &str) -> String { + #[cfg(test)] + { + if let Some(src) = MOCK_INTEGRATION_BUNDLES + .read() + .expect("mock bundle lock should not be poisoned") + .get(name) + .cloned() + { + return src; + } + } + script_src(&integration_bundle_filename(name)).unwrap_or_else(|| integration_fallback_src(name)) +} + +/// Returns a `", src) + } else { + format!("", src, attrs) + } +} + +#[cfg(test)] +pub fn mock_integration_bundle(name: &str, src: impl Into) { + MOCK_INTEGRATION_BUNDLES + .write() + .expect("mock bundle lock should not be poisoned") + .insert(name.to_string(), src.into()); +} + +#[cfg(test)] +pub fn clear_mock_integration_bundles() { + MOCK_INTEGRATION_BUNDLES + .write() + .expect("mock bundle lock should not be poisoned") + .clear(); +} diff --git a/crates/js/lib/src/integrations/starlight.ts b/crates/js/lib/src/integrations/testlight.ts similarity index 53% rename from crates/js/lib/src/integrations/starlight.ts rename to crates/js/lib/src/integrations/testlight.ts index cc1a0db..6920b1e 100644 --- a/crates/js/lib/src/integrations/starlight.ts +++ b/crates/js/lib/src/integrations/testlight.ts @@ -3,17 +3,17 @@ import { installQueue } from '../core/queue'; import { log } from '../core/log'; import { resolvePrebidWindow, PrebidWindow } from '../shared/globals'; -type StarlightCallback = () => void; +type TestlightCallback = () => void; -type StarlightGlobal = { - que?: StarlightCallback[]; +type TestlightGlobal = { + que?: TestlightCallback[]; }; -type StarlightWindow = PrebidWindow & { - starlight?: StarlightGlobal; +type TestlightWindow = PrebidWindow & { + testlight?: TestlightGlobal; }; -function ensureTsjsApi(win: StarlightWindow): TsjsApi { +function ensureTsjsApi(win: TestlightWindow): TsjsApi { if (win.tsjs) return win.tsjs; const stub: TsjsApi = { version: '0.0.0', @@ -26,13 +26,13 @@ function ensureTsjsApi(win: StarlightWindow): TsjsApi { return stub; } -function installStarlightQueue(api: TsjsApi, win: StarlightWindow): void { +function installTestlightQueue(api: TsjsApi, win: TestlightWindow): void { if (!Array.isArray(api.que)) { installQueue(api, win); } } -function flushCallbacks(queue: StarlightCallback[], api: TsjsApi): void { +function flushCallbacks(queue: TestlightCallback[], api: TsjsApi): void { while (queue.length > 0) { const fn = queue.shift(); if (typeof fn !== 'function') { @@ -44,25 +44,25 @@ function flushCallbacks(queue: StarlightCallback[], api: TsjsApi): void { } else { fn.call(api); } - log.debug('starlight shim: flushed callback'); + log.debug('testlight shim: flushed callback'); } catch (err) { - log.debug('starlight shim: queued callback threw', err); + log.debug('testlight shim: queued callback threw', err); } } } -export function installStarlightShim(): boolean { - const win = resolvePrebidWindow() as StarlightWindow; +export function installTestlightShim(): boolean { + const win = resolvePrebidWindow() as TestlightWindow; const api = ensureTsjsApi(win); - installStarlightQueue(api, win); + installTestlightQueue(api, win); - const starlight = (win.starlight = win.starlight ?? {}); - const pending: StarlightCallback[] = Array.isArray(starlight.que) ? [...starlight.que] : []; - const queue: StarlightCallback[] = []; - starlight.que = queue; + const testlight = (win.testlight = win.testlight ?? {}); + const pending: TestlightCallback[] = Array.isArray(testlight.que) ? [...testlight.que] : []; + const queue: TestlightCallback[] = []; + testlight.que = queue; const originalPush = queue.push.bind(queue); - queue.push = function (...callbacks: StarlightCallback[]): number { + queue.push = function (...callbacks: TestlightCallback[]): number { const len = originalPush(...callbacks); flushCallbacks(queue, api); return len; @@ -72,10 +72,10 @@ export function installStarlightShim(): boolean { queue.push(...pending); } - log.info('starlight shim installed', { queuedCallbacks: queue.length }); + log.info('testlight shim installed', { queuedCallbacks: queue.length }); return true; } if (typeof window !== 'undefined') { - installStarlightShim(); + installTestlightShim(); } diff --git a/docs/integration_guide.md b/docs/integration_guide.md index 0fe99b1..c364e6b 100644 --- a/docs/integration_guide.md +++ b/docs/integration_guide.md @@ -1,17 +1,17 @@ # Integration Guide This document explains how to integrate a new integration module with the Trusted Server -runtime. The workflow mirrors the built‑in `starlight` sample in -`crates/common/src/integrations/starlight.rs`. +runtime. The workflow mirrors the built‑in `testlight` sample in +`crates/common/src/integrations/testlight.rs`. ## Architecture Overview -| Component | Purpose | -| --- | --- | -| `crates/common/src/integrations/registry.rs` | Defines the `IntegrationProxy` and `IntegrationAttributeRewriter` traits and hosts the `IntegrationRegistry`, which drives proxy routing and HTML rewrites. | -| `Settings::integrations` (`crates/common/src/settings.rs`) | Free‑form JSON blob keyed by integration ID. Each module deserializes its own config so the core settings schema stays stable. | -| Fastly entrypoint (`crates/fastly/src/main.rs`) | Instantiates the registry once per request, routes `/integrations//…` requests to the appropriate proxy, and passes the registry to the publisher origin proxy so HTML rewriting remains integration-aware. | -| `html_processor.rs` | Applies first‑party URL rewrites, injects the Trusted Server JS shim, and lets integrations override attribute values (for example to swap script URLs). | +| Component | Purpose | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `crates/common/src/integrations/registry.rs` | Defines the `IntegrationProxy`, `IntegrationAttributeRewriter`, and `IntegrationScriptRewriter` traits and hosts the `IntegrationRegistry`, which drives proxy routing and HTML/text rewrites. | +| `Settings::integrations` (`crates/common/src/settings.rs`) | Free‑form JSON blob keyed by integration ID. Use `IntegrationSettings::insert_config` to seed configs; each module deserializes and validates (`validator::Validate`) its own config and exposes an `enabled` flag so the core settings schema stays stable. | +| Fastly entrypoint (`crates/fastly/src/main.rs`) | Instantiates the registry once per request, routes `/integrations//…` requests to the appropriate proxy, and passes the registry to the publisher origin proxy so HTML rewriting remains integration-aware. | +| `html_processor.rs` | Applies first‑party URL rewrites, injects the Trusted Server JS shim, and lets integrations override attribute values (for example to swap script URLs). | ## Step-by-Step Integration @@ -31,27 +31,81 @@ rewrite_scripts = true ### 2. Create the integration module Add a module under `crates/common/src/integrations//mod.rs` (see -`crates/common/src/integrations/starlight.rs` for reference) and expose it in +`crates/common/src/integrations/testlight.rs` for reference) and expose it in `crates/common/src/integrations/mod.rs`. Key pieces: ```rust -#[derive(Deserialize)] -struct MyIntegrationConfig { /* … */ } +#[derive(Deserialize, Validate)] +struct MyIntegrationConfig { + #[serde(default = "default_enabled")] + enabled: bool, + // … +} + +impl IntegrationConfig for MyIntegrationConfig { + fn is_enabled(&self) -> bool { self.enabled } +} pub struct MyIntegration { config: MyIntegrationConfig, } pub fn build(settings: &Settings) -> Option> { - let raw = settings.integration_config("my_integration")?; - let config: MyIntegrationConfig = serde_json::from_value(raw.clone()).ok()?; + let config = settings + .integration_config::("my_integration") + .ok() + .flatten()?; Some(Arc::new(MyIntegration { config })) } + +// Tests or scaffolding code can seed configs without hand-writing JSON: +settings + .integrations + .insert_config( + "my_integration", + &serde_json::json!({ + "enabled": true, + "endpoint": "https://example.com/api" + }), + )?; +``` + +`Settings::integration_config::` automatically deserializes the raw JSON blob, +runs [`validator`](https://docs.rs/validator/latest/validator/) on the type, and +drops configs whose `is_enabled` returns `false`. Always derive/implement +`Validate` for schema enforcement and implement `IntegrationConfig` (typically +wrapping a `#[serde(default)] enabled` flag) so operators can toggle +integrations without code changes. + +### 3. Return an `IntegrationRegistration` + +Each integration registers itself via a `register` function that returns an +`IntegrationRegistration`. This object describes which HTTP proxies and HTML +rewrites the integration exposes: + +```rust +pub fn register(settings: &Settings) -> Option { + let integration = build(settings)?; + Some( + IntegrationRegistration::builder("my_integration") + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()) + .with_script_rewriter(integration) + .with_asset("my_integration") + .build(), + ) +} ``` -### 3. Implement `IntegrationProxy` for endpoints +Any combination of the three vectors may be populated. Modules that only need +HTML rewrites can skip the `proxies` field altogether, and vice versa. The +registry automatically iterates over the static builder list in +`crates/common/src/integrations/mod.rs`, so adding the new `register` function +is enough to make the integration discoverable. + +### 4. Implement `IntegrationProxy` for endpoints Implement the trait from `registry.rs` when your integration needs its own HTTP entrypoint: @@ -81,10 +135,11 @@ already injects Trusted Server logging, headers, and error handling; the handler needs to deserialize the request, call the upstream endpoint, and stamp integration-specific headers. -### 4. Implement `IntegrationAttributeRewriter` for shims (optional) +### 5. Implement HTML rewrite hooks (optional) -If the integration needs to rewrite script/link tags or inject HTML, implement the -`IntegrationAttributeRewriter` trait: +If the integration needs to rewrite script/link tags or inject HTML, implement +`IntegrationAttributeRewriter` for attribute mutation and +`IntegrationScriptRewriter` for inline ` "#; let mut config = create_test_config(); - config.enable_prebid = true; // enable rewriting of Prebid URLs + config.enable_prebid = true; // enable removal of Prebid URLs let processor = create_html_processor(config); let pipeline_config = PipelineConfig { input_compression: Compression::None, @@ -489,19 +435,20 @@ mod tests { let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); assert!(result.is_ok()); let processed = String::from_utf8_lossy(&output); - assert!(processed.contains("/static/tsjs=tsjs-core.min.js")); - // Prebid references are rewritten to our extension when auto-configure is on - assert!(processed.contains("/static/tsjs=tsjs-ext.min.js")); + assert!(processed.contains("/static/tsjs=tsjs-unified.min.js")); + // Prebid script references should be removed when auto-configure is on + assert!(!processed.contains("prebid.min.js")); + assert!(!processed.contains("cdn.prebid.org/prebid.js")); } #[test] - fn test_injects_tsjs_script_and_rewrites_prebid_with_query_string() { + fn test_injects_unified_bundle_and_removes_prebid_with_query_string() { let html = r#" "#; let mut config = create_test_config(); - config.enable_prebid = true; // enable rewriting of Prebid URLs + config.enable_prebid = true; // enable removal of Prebid URLs let processor = create_html_processor(config); let pipeline_config = PipelineConfig { input_compression: Compression::None, @@ -514,19 +461,21 @@ mod tests { let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); assert!(result.is_ok()); let processed = String::from_utf8_lossy(&output); - assert!(processed.contains("/static/tsjs=tsjs-core.min.js")); - assert!(processed.contains("/static/tsjs=tsjs-ext.min.js")); + // Should inject unified bundle + assert!(processed.contains("/static/tsjs=tsjs-unified.min.js")); + // Prebid script should be removed + assert!(!processed.contains("prebidjs.min.js")); } #[test] - fn test_always_injects_tsjs_script() { + fn test_always_injects_unified_bundle() { let html = r#" "#; let mut config = create_test_config(); - config.enable_prebid = false; // No longer affects tsjs injection + config.enable_prebid = false; // When disabled, don't remove Prebid scripts let processor = create_html_processor(config); let pipeline_config = PipelineConfig { input_compression: Compression::None, @@ -539,10 +488,11 @@ mod tests { let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); assert!(result.is_ok()); let processed = String::from_utf8_lossy(&output); - // When auto-configure is disabled, do not rewrite Prebid references + // When auto-configure is disabled, do not remove Prebid references assert!(processed.contains("/js/prebid.min.js")); assert!(processed.contains("cdn.prebid.org/prebid.js")); - assert!(processed.contains("/static/tsjs=tsjs-core.min.js")); + // But still inject unified bundle + assert!(processed.contains("/static/tsjs=tsjs-unified.min.js")); } #[test] @@ -712,10 +662,10 @@ mod tests { use crate::test_support::tests::create_test_settings; let settings = create_test_settings(); - let registry = IntegrationRegistry::new(&settings); + let integrations = IntegrationRegistry::default(); let config = HtmlProcessorConfig::from_settings( &settings, - ®istry, + &integrations, "origin.test-publisher.com", "proxy.example.com", "https", @@ -808,58 +758,6 @@ mod tests { ); } - #[test] - fn test_integration_registry_rewrites_integration_scripts() { - use serde_json::json; - - let html = r#" - - "#; - - let _bundle_guard = mock_testlight_bundle(); - let mut settings = Settings::default(); - let shim_src = tsjs::integration_script_src("testlight"); - settings - .integrations - .insert_config( - "testlight", - &json!({ - "enabled": true, - "endpoint": "https://example.com/openrtb2/auction", - "rewrite_scripts": true, - "shim_src": shim_src, - }), - ) - .expect("should insert testlight config"); - - let registry = IntegrationRegistry::new(&settings); - let mut config = create_test_config(); - config.integrations = registry; - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); - assert!(result.is_ok()); - - let processed = String::from_utf8_lossy(&output); - let expected_src = tsjs::integration_script_src("testlight"); - assert!( - processed.contains(&expected_src), - "Integration shim should replace integration script reference" - ); - assert!( - !processed.contains("cdn.testlight.com"), - "Original integration URL should be removed" - ); - } - #[test] fn test_real_publisher_html_with_gzip() { use flate2::read::GzDecoder; diff --git a/crates/common/src/integrations/registry.rs b/crates/common/src/integrations/registry.rs index 762ec83..f3ea428 100644 --- a/crates/common/src/integrations/registry.rs +++ b/crates/common/src/integrations/registry.rs @@ -103,7 +103,6 @@ pub struct IntegrationRegistration { pub proxies: Vec>, pub attribute_rewriters: Vec>, pub script_rewriters: Vec>, - pub assets: Vec, } impl IntegrationRegistration { @@ -125,7 +124,6 @@ impl IntegrationRegistrationBuilder { proxies: Vec::new(), attribute_rewriters: Vec::new(), script_rewriters: Vec::new(), - assets: Vec::new(), }, } } @@ -151,12 +149,6 @@ impl IntegrationRegistrationBuilder { self } - #[must_use] - pub fn with_asset(mut self, asset: impl Into) -> Self { - self.registration.assets.push(asset.into()); - self - } - #[must_use] pub fn build(self) -> IntegrationRegistration { self.registration @@ -172,7 +164,6 @@ struct IntegrationRegistryInner { routes: Vec<(IntegrationEndpoint, &'static str)>, html_rewriters: Vec>, script_rewriters: Vec>, - assets: Vec<(&'static str, String)>, } /// Summary of registered integration capabilities. @@ -182,7 +173,6 @@ pub struct IntegrationMetadata { pub routes: Vec, pub attribute_rewriters: usize, pub script_selectors: Vec<&'static str>, - pub assets: Vec, } impl IntegrationMetadata { @@ -192,7 +182,6 @@ impl IntegrationMetadata { routes: Vec::new(), attribute_rewriters: 0, script_selectors: Vec::new(), - assets: Vec::new(), } } } @@ -234,12 +223,6 @@ impl IntegrationRegistry { inner .script_rewriters .extend(registration.script_rewriters.into_iter()); - inner.assets.extend( - registration - .assets - .into_iter() - .map(|asset| (registration.integration_id, asset)), - ); } } @@ -332,13 +315,6 @@ impl IntegrationRegistry { entry.script_selectors.push(rewriter.selector()); } - for (integration_id, asset) in &self.inner.assets { - let entry = map - .entry(*integration_id) - .or_insert_with(|| IntegrationMetadata::new(integration_id)); - entry.assets.push(asset.clone()); - } - map.into_values().collect() } } diff --git a/crates/common/src/integrations/testlight.rs b/crates/common/src/integrations/testlight.rs index a216fd4..a506ac6 100644 --- a/crates/common/src/integrations/testlight.rs +++ b/crates/common/src/integrations/testlight.rs @@ -115,7 +115,6 @@ pub fn register(settings: &Settings) -> Option { IntegrationRegistration::builder(TESTLIGHT_INTEGRATION_ID) .with_proxy(integration.clone()) .with_attribute_rewriter(integration) - .with_asset("testlight") .build(), ) } @@ -201,6 +200,10 @@ impl IntegrationAttributeRewriter for TestlightIntegration { let lowered = attr_value.to_ascii_lowercase(); if lowered.contains("testlight.js") { + // TODO: need a way to remove the whole script tag + // None will still load external script Some will only rewrite attr + // but testlight script is now backed into the unified build + // for now this is loading the unified js again. Some(self.config.shim_src.clone()) } else { None @@ -213,7 +216,8 @@ fn default_timeout_ms() -> u32 { } fn default_shim_src() -> String { - tsjs::integration_script_src("testlight") + // Testlight is included in the unified bundle, so we return the unified script source + tsjs::unified_script_src() } fn default_enabled() -> bool { @@ -243,21 +247,6 @@ mod tests { use fastly::http::Method; use serde_json::json; - const MOCK_TESTLIGHT_SRC: &str = "https://mock.testassets/testlight.js"; - - struct MockBundleGuard; - - fn mock_testlight_bundle() -> MockBundleGuard { - tsjs::mock_integration_bundle("testlight", MOCK_TESTLIGHT_SRC); - MockBundleGuard - } - - impl Drop for MockBundleGuard { - fn drop(&mut self) { - tsjs::clear_mock_integration_bundles(); - } - } - #[test] fn build_requires_config() { let settings = create_test_settings(); @@ -269,8 +258,7 @@ mod tests { #[test] fn html_rewriter_replaces_integration_script() { - let _bundle_guard = mock_testlight_bundle(); - let shim_src = tsjs::integration_script_src("testlight"); + let shim_src = tsjs::unified_script_src(); let config = TestlightConfig { enabled: true, endpoint: "https://example.com/openrtb".to_string(), @@ -298,8 +286,7 @@ mod tests { #[test] fn html_rewriter_is_noop_when_disabled() { - let _bundle_guard = mock_testlight_bundle(); - let shim_src = tsjs::integration_script_src("testlight"); + let shim_src = tsjs::unified_script_src(); let config = TestlightConfig { enabled: true, endpoint: "https://example.com/openrtb".to_string(), diff --git a/crates/common/src/tsjs.rs b/crates/common/src/tsjs.rs index ffb1e68..26690da 100644 --- a/crates/common/src/tsjs.rs +++ b/crates/common/src/tsjs.rs @@ -1,26 +1,14 @@ -use trusted_server_js::{bundle_for_filename, bundle_hash}; +use trusted_server_js::{bundle_hash, TsjsBundle}; -#[cfg(test)] -use once_cell::sync::Lazy; -#[cfg(test)] -use std::collections::HashMap; -#[cfg(test)] -use std::sync::RwLock; - -#[cfg(test)] -static MOCK_INTEGRATION_BUNDLES: Lazy>> = - Lazy::new(|| RwLock::new(HashMap::new())); - -fn script_src_internal(filename: &str) -> String { - script_src(filename).unwrap_or_else(|| { - panic!( - "tsjs: bundle {} not found. Ensure npm build produced this bundle.", - filename - ) - }) +fn script_src_for(bundle: TsjsBundle) -> String { + format!( + "/static/tsjs={}?v={}", + bundle.minified_filename(), + bundle_hash(bundle) + ) } -fn script_tag_internal(filename: &str, attrs: &str) -> String { +fn script_tag_for(bundle: TsjsBundle, attrs: &str) -> String { let attr_segment = if attrs.is_empty() { String::new() } else { @@ -28,107 +16,17 @@ fn script_tag_internal(filename: &str, attrs: &str) -> String { }; format!( "", - script_src_internal(filename), + script_src_for(bundle), attr_segment ) } -/// Returns `/static/tsjs=?v=` if the bundle is available. -pub fn script_src(filename: &str) -> Option { - let trimmed = filename.trim(); - bundle_for_filename(trimmed)?; - let hash = bundle_hash(trimmed)?; - let minified = trimmed - .strip_suffix(".js") - .map(|stem| format!("{stem}.min.js")) - .unwrap_or_else(|| format!("{trimmed}.min.js")); - Some(format!("/static/tsjs={}?v={}", minified, hash)) -} - -/// Returns a `", src) - } else { - format!("", src, attrs) - } - }) -} - -/// `", src) - } else { - format!("", src, attrs) - } -} - -#[cfg(test)] -pub fn mock_integration_bundle(name: &str, src: impl Into) { - MOCK_INTEGRATION_BUNDLES - .write() - .expect("mock bundle lock should not be poisoned") - .insert(name.to_string(), src.into()); +/// `/static` URL for the unified bundle with cache-busting hash. +pub fn unified_script_src() -> String { + script_src_for(TsjsBundle::Unified) } -#[cfg(test)] -pub fn clear_mock_integration_bundles() { - MOCK_INTEGRATION_BUNDLES - .write() - .expect("mock bundle lock should not be poisoned") - .clear(); +/// ` - // optionally load Prebid shim when pbjs is present - // - -``` - -## Auto‑Rewrite (Server) - -- When auto-configure is enabled, the HTML processor injects the core loader and rewrites any Prebid script URLs to `/static/tsjs=tsjs-ext.min.js`. The extension aliases `window.pbjs` to `window.tsjs` and flushes `pbjs.que`. -- Proxied creative HTML injects the creative helper once at the top of ``: `/static/tsjs=tsjs-creative.min.js`. The helper monitors anchors for script-driven rewrites and rebuilds first-party click URLs whenever creatives mutate them. - -## First-Party Proxy Flows - -The Rust services (`trusted-server-common`) expose several proxy entry points that work together to keep all ad traffic on the publisher’s domain while propagating the synthetic identifier generated for the user. - -### Publisher Origin Proxy - -- Endpoint: `handle_publisher_request` (`crates/common/src/publisher.rs`). -- Retrieves or generates the trusted synthetic identifier before Fastly consumes the request body. -- Always stamps the proxied response with `X-Synthetic-Fresh` and `x-psid-ts` headers and, when the browser does not already present one, sets the `synthetic_id=` cookie (Secure + SameSite=Lax) bound to the configured publisher domain. -- Result: downstream assets fetched through the same first-party origin automatically include the synthetic ID header/cookie so subsequent proxy layers can read it. - -### Creative Asset Proxy - -- Endpoint: `handle_first_party_proxy` (`crates/common/src/proxy.rs`). -- Accepts the signed `/first-party/proxy?tsurl=...` URLs injected by the HTML rewriter and streams the creative from the third-party origin. -- Extracts the synthetic ID from the inbound cookie or header and forwards it to the creative origin by appending `synthetic_id=` to the rewritten target URL (while preserving existing query parameters). -- Follows HTTP redirects (301/302/303/307/308) up to four hops, re-validating each `Location`, switching to `GET` after 303 responses, and propagating the synthetic ID on every hop. -- Ensures the response body is rewritten when it is HTML/CSS/JS so all nested asset requests loop back through the same first-party proxy. - -### Click-Through Proxy - -- Endpoint: `handle_first_party_click` (`crates/common/src/proxy.rs`). -- Validates the signed `/first-party/click` URL generated for anchors inside proxied creatives. -- On success, issues an HTTP 302 to the reconstructed destination and appends `synthetic_id=` if the user presented one, letting downstream measurement end points correlate the click with the original synthetic identifier. -- Ensures click responses are never cached (`Cache-Control: no-store, private`). - -Together these layers guarantee that the synthetic identifier generated on the publisher response is preserved throughout page loads, asset fetches, and click-throughs without exposing the third-party origins directly to the browser. - -## Notes - -- By default, the build fails if `tsjs-core.js` cannot be produced. To change behavior: - - `TSJS_SKIP_BUILD=1`: skip running npm; requires `dist/tsjs-core.js` to exist so it can be copied to `OUT_DIR`. - - `TSJS_ALLOW_FALLBACK=1`: allow using a checked‑in `dist/tsjs-core.js` if the npm build didn’t produce an output. - - `TSJS_TEST=1`: run `npm test` during the build. diff --git a/crates/js/build.rs b/crates/js/build.rs index e45abff..ba3fe57 100644 --- a/crates/js/build.rs +++ b/crates/js/build.rs @@ -1,10 +1,9 @@ use std::env; -use std::fs::{self, File}; -use std::io::Write; +use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -const REQUIRED_BUNDLES: &[&str] = &["tsjs-core.js", "tsjs-ext.js", "tsjs-creative.js"]; +const UNIFIED_BUNDLE: &str = "tsjs-unified.js"; fn main() { // Rebuild if TS sources change (belt-and-suspenders): enumerate every file under ts/ @@ -61,11 +60,15 @@ fn main() { .status(); } - // Build bundle + // Build unified bundle if !skip { if let Some(npm_path) = npm.clone() { - let status = Command::new(npm_path) - .args(["run", "build"]) + println!("cargo:warning=tsjs: Building unified bundle"); + let js_modules = env::var("TSJS_MODULES").unwrap_or("".to_string()); + + let status = Command::new(&npm_path) + .env("TSJS_MODULES", js_modules) + .args(["run", "build:custom"]) .current_dir(&ts_dir) .status(); if !status.as_ref().map(|s| s.success()).unwrap_or(false) { @@ -74,29 +77,40 @@ fn main() { } } - // Copy the result into OUT_DIR for include_str! - let bundle_files = discover_bundles(&dist_dir); - ensure_required_bundles(&bundle_files); - copy_bundles(&bundle_files, &dist_dir, &out_dir); - generate_manifest(&bundle_files, &out_dir); + // Copy unified bundle into OUT_DIR for include_str! + copy_bundle(UNIFIED_BUNDLE, true, &crate_dir, &dist_dir, &out_dir); } -fn discover_bundles(dist_dir: &Path) -> Vec { - let mut bundles = Vec::new(); - let entries = match fs::read_dir(dist_dir) { - Ok(entries) => entries, - Err(_) => return bundles, - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) == Some("js") { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - bundles.push(name.to_string()); +fn copy_bundle( + filename: &str, + required: bool, + crate_dir: &Path, + dist_dir: &Path, + out_dir: &Path, +) { + let primary = dist_dir.join(filename); + let fallback = crate_dir.join("dist").join(filename); + let target = out_dir.join(filename); + + for source in [&primary, &fallback] { + if source.exists() { + if let Err(e) = fs::copy(source, &target) { + if required { + panic!("tsjs: failed to copy {:?} to {:?}: {}", source, target, e); + } } + return; } } - bundles.sort(); - bundles + + if required { + panic!( + "tsjs: bundle {} not found: {:?} (and fallback {:?}). Ensure Node is installed and `npm run build` succeeds, or commit dist/{}.", + filename, primary, fallback, filename + ); + } + + let _ = fs::write(&target, ""); } fn watch_dir_recursively(root: &Path) { @@ -121,40 +135,3 @@ fn watch_dir_recursively(root: &Path) { } } } - -fn ensure_required_bundles(bundles: &[String]) { - for required in REQUIRED_BUNDLES { - if !bundles.iter().any(|bundle| bundle == required) { - panic!("tsjs: required bundle {} not found in dist/", required); - } - } -} - -fn copy_bundles(bundles: &[String], dist_dir: &Path, out_dir: &Path) { - for bundle in bundles { - let source = dist_dir.join(bundle); - let target = out_dir.join(bundle); - if let Err(e) = fs::copy(&source, &target) { - panic!( - "tsjs: failed to copy bundle {:?} to {:?}: {}", - source, target, e - ); - } - } -} - -fn generate_manifest(bundles: &[String], out_dir: &Path) { - let manifest_path = out_dir.join("bundle_manifest.rs"); - let mut file = File::create(&manifest_path) - .unwrap_or_else(|e| panic!("tsjs: failed to create manifest: {}", e)); - writeln!(&mut file, "pub const BUNDLES: &[(&str, &str)] = &[").unwrap(); - for bundle in bundles { - writeln!( - &mut file, - " (\"{name}\", include_str!(concat!(env!(\"OUT_DIR\"), \"/{name}\"))),", - name = bundle - ) - .unwrap(); - } - writeln!(&mut file, "];").unwrap(); -} diff --git a/crates/js/lib/.gitignore b/crates/js/lib/.gitignore new file mode 100644 index 0000000..f1283e5 --- /dev/null +++ b/crates/js/lib/.gitignore @@ -0,0 +1,2 @@ +# Auto-generated files +src/generated-modules.ts diff --git a/crates/js/lib/package-lock.json b/crates/js/lib/package-lock.json index 6e0c2d6..33c2aa4 100644 --- a/crates/js/lib/package-lock.json +++ b/crates/js/lib/package-lock.json @@ -166,7 +166,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -210,7 +209,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -1409,7 +1407,6 @@ "integrity": "sha512-tK3GPFWbirvNgsNKto+UmB/cRtn6TZfyw0D6IKrW55n6Vbs7KJoZtI//kpTKzE/DUmmnAFD8/Ca46s7Obs92/w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.4", "@typescript-eslint/types": "8.46.4", @@ -1726,7 +1723,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2032,7 +2028,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", @@ -2720,7 +2715,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -5529,7 +5523,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -5746,7 +5739,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -5945,7 +5937,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/crates/js/lib/package.json b/crates/js/lib/package.json index 5538f45..93164d3 100644 --- a/crates/js/lib/package.json +++ b/crates/js/lib/package.json @@ -5,8 +5,9 @@ "type": "module", "description": "Trusted Server tsjs TypeScript library with queue and simple banner rendering.", "scripts": { - "build": "node scripts/build-all.mjs", - "dev": "node scripts/watch-all.mjs", + "build": "vite build", + "build:custom": "vite build", + "dev": "vite build --watch", "test": "vitest run", "test:watch": "vitest", "lint": "eslint \"src/**/*.{ts,tsx}\"", diff --git a/crates/js/lib/scripts/build-all.mjs b/crates/js/lib/scripts/build-all.mjs deleted file mode 100644 index f327680..0000000 --- a/crates/js/lib/scripts/build-all.mjs +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node - -import { spawnSync } from 'node:child_process'; -import { readdirSync, existsSync } from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const projectRoot = path.resolve(__dirname, '..'); -const integrationsDir = path.resolve(projectRoot, 'src', 'integrations'); - -function discoverIntegrationBundles() { - if (!existsSync(integrationsDir)) { - return []; - } - return readdirSync(integrationsDir, { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.endsWith('.ts')) - .map((entry) => entry.name.replace(/\.ts$/i, '')); -} - -function runBundle(key) { - const env = { ...process.env, TSJS_BUNDLE: key }; - const result = spawnSync('npx', ['vite', 'build'], { - cwd: projectRoot, - env, - stdio: 'inherit', - }); - if (result.status !== 0) { - process.exit(result.status ?? 1); - } -} - -const baseBundles = ['core', 'ext', 'creative']; -const integrationBundles = discoverIntegrationBundles().map((name) => `integration:${name}`); -const bundles = [...baseBundles, ...integrationBundles]; - -if (bundles.length === 0) { - console.warn('tsjs: no bundles discovered; skipping build'); - process.exit(0); -} - -for (const bundle of bundles) { - console.log(`tsjs: building bundle "${bundle}"`); - runBundle(bundle); -} diff --git a/crates/js/lib/scripts/watch-all.mjs b/crates/js/lib/scripts/watch-all.mjs deleted file mode 100755 index 6434afe..0000000 --- a/crates/js/lib/scripts/watch-all.mjs +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env node -import { spawn } from 'node:child_process'; - -const modes = ['core', 'ext', 'creative']; -const children = []; - -function spawnWatcher(mode) { - const child = spawn('npx', ['vite', 'build', '--mode', mode, '--watch'], { - stdio: 'inherit', - env: process.env, - }); - child.on('exit', (code, signal) => { - if (signal) { - console.log(`[watch] mode ${mode} exited via signal ${signal}`); - } else { - console.log(`[watch] mode ${mode} exited with code ${code}`); - } - }); - children.push(child); -} - -modes.forEach(spawnWatcher); - -function shutdown(signal) { - console.log(`\n[watch] received ${signal}; shutting down watchers...`); - for (const child of children) { - if (!child.killed) child.kill('SIGTERM'); - } - process.exit(0); -} - -process.on('SIGINT', () => shutdown('SIGINT')); -process.on('SIGTERM', () => shutdown('SIGTERM')); diff --git a/crates/js/lib/src/index.ts b/crates/js/lib/src/index.ts new file mode 100644 index 0000000..7cd2a5e --- /dev/null +++ b/crates/js/lib/src/index.ts @@ -0,0 +1,48 @@ +// Unified tsjs bundle entry point +// This file conditionally imports modules based on build-time configuration +import type { TsjsApi } from './core/types'; +import { modules, type ModuleName } from './generated-modules'; +import { log } from './core/log'; + +const VERSION = '0.1.0-unified'; + +// Ensure we have a window object +const w: Window & { tsjs?: TsjsApi } = + ((globalThis as unknown as { window?: Window }).window as Window & { + tsjs?: TsjsApi; + }) || ({} as Window & { tsjs?: TsjsApi }); + +// Log which modules are included in this build +const includedModules = Object.keys(modules) as ModuleName[]; +log.info('tsjs unified bundle initialized', { + version: VERSION, + modules: includedModules, +}); + +// The core module sets up the main API and should always be included +// If core is included, it will have already initialized the tsjs global +// and set up the queue system + +// Initialize optional modules if they're included +for (const [moduleName, moduleExports] of Object.entries(modules)) { + if (moduleName === 'core') { + // Core is already initialized via its own IIFE-style init code + continue; + } + + // For other modules, check if they have an init function or are self-initializing + if (typeof moduleExports === 'object' && moduleExports !== null) { + // Log that the module is available + log.debug(`tsjs: module '${moduleName}' loaded`); + + // Some modules like 'ext' are self-initializing (they run on import) + // Some modules like 'creative' export an API object + // We don't need to do anything special here - just importing them is enough + } +} + +// Re-export core types for convenience +export type { AdUnit, TsjsApi } from './core/types'; + +// Export the modules object for advanced use cases +export { modules }; diff --git a/crates/js/lib/src/creative/click.ts b/crates/js/lib/src/integrations/creative/click.ts similarity index 98% rename from crates/js/lib/src/creative/click.ts rename to crates/js/lib/src/integrations/creative/click.ts index dc5289a..a505c03 100644 --- a/crates/js/lib/src/creative/click.ts +++ b/crates/js/lib/src/integrations/creative/click.ts @@ -1,8 +1,8 @@ // Click guard runtime: detects mutated tracking URLs and rebuilds signed first-party clicks. -import { log } from '../core/log'; -import { creativeGlobal } from '../shared/globals'; -import { delay, queueTask } from '../shared/async'; -import { createMutationScheduler } from '../shared/scheduler'; +import { log } from '../../core/log'; +import { creativeGlobal } from '../../shared/globals'; +import { delay, queueTask } from '../../shared/async'; +import { createMutationScheduler } from '../../shared/scheduler'; type AnchorLike = HTMLAnchorElement | HTMLAreaElement; type Canon = { base: string; params: Record }; diff --git a/crates/js/lib/src/creative/dynamic_src_guard.ts b/crates/js/lib/src/integrations/creative/dynamic_src_guard.ts similarity index 99% rename from crates/js/lib/src/creative/dynamic_src_guard.ts rename to crates/js/lib/src/integrations/creative/dynamic_src_guard.ts index 569e0ef..2fc216c 100644 --- a/crates/js/lib/src/creative/dynamic_src_guard.ts +++ b/crates/js/lib/src/integrations/creative/dynamic_src_guard.ts @@ -1,5 +1,5 @@ -import { log } from '../core/log'; -import { createMutationScheduler } from '../shared/scheduler'; +import { log } from '../../core/log'; +import { createMutationScheduler } from '../../shared/scheduler'; type ElementWithSrc = Element & { src: string }; diff --git a/crates/js/lib/src/creative/iframe.ts b/crates/js/lib/src/integrations/creative/iframe.ts similarity index 100% rename from crates/js/lib/src/creative/iframe.ts rename to crates/js/lib/src/integrations/creative/iframe.ts diff --git a/crates/js/lib/src/creative/image.ts b/crates/js/lib/src/integrations/creative/image.ts similarity index 100% rename from crates/js/lib/src/creative/image.ts rename to crates/js/lib/src/integrations/creative/image.ts diff --git a/crates/js/lib/src/creative/index.ts b/crates/js/lib/src/integrations/creative/index.ts similarity index 95% rename from crates/js/lib/src/creative/index.ts rename to crates/js/lib/src/integrations/creative/index.ts index 2f16bbb..3955535 100644 --- a/crates/js/lib/src/creative/index.ts +++ b/crates/js/lib/src/integrations/creative/index.ts @@ -1,7 +1,7 @@ // Entry point for the creative runtime: wires up click + image + iframe guards globally. -import { log } from '../core/log'; -import type { TsCreativeConfig, CreativeWindow, TsCreativeApi } from '../shared/globals'; -import { creativeGlobal, resolveWindow } from '../shared/globals'; +import { log } from '../../core/log'; +import type { TsCreativeConfig, CreativeWindow, TsCreativeApi } from '../../shared/globals'; +import { creativeGlobal, resolveWindow } from '../../shared/globals'; import { installClickGuard } from './click'; import { installDynamicImageProxy } from './image'; diff --git a/crates/js/lib/src/creative/proxy_sign.ts b/crates/js/lib/src/integrations/creative/proxy_sign.ts similarity index 97% rename from crates/js/lib/src/creative/proxy_sign.ts rename to crates/js/lib/src/integrations/creative/proxy_sign.ts index fc89acc..b18d7d1 100644 --- a/crates/js/lib/src/creative/proxy_sign.ts +++ b/crates/js/lib/src/integrations/creative/proxy_sign.ts @@ -1,4 +1,4 @@ -import { log } from '../core/log'; +import { log } from '../../core/log'; const PROXY_PREFIX = '/first-party/proxy'; diff --git a/crates/js/lib/src/ext/index.ts b/crates/js/lib/src/integrations/ext/index.ts similarity index 100% rename from crates/js/lib/src/ext/index.ts rename to crates/js/lib/src/integrations/ext/index.ts diff --git a/crates/js/lib/src/ext/prebidjs.ts b/crates/js/lib/src/integrations/ext/prebidjs.ts similarity index 92% rename from crates/js/lib/src/ext/prebidjs.ts rename to crates/js/lib/src/integrations/ext/prebidjs.ts index 4d6224c..00837f7 100644 --- a/crates/js/lib/src/ext/prebidjs.ts +++ b/crates/js/lib/src/integrations/ext/prebidjs.ts @@ -1,9 +1,9 @@ // Prebid.js compatibility shim: exposes tsjs API through the legacy pbjs global. -import type { TsjsApi, HighestCpmBid, RequestAdsCallback, RequestAdsOptions } from '../core/types'; -import { log } from '../core/log'; -import { installQueue } from '../core/queue'; -import { getAllCodes, getAllUnits, firstSize } from '../core/registry'; -import { resolvePrebidWindow, PrebidWindow } from '../shared/globals'; +import type { TsjsApi, HighestCpmBid, RequestAdsCallback, RequestAdsOptions } from '../../core/types'; +import { log } from '../../core/log'; +import { installQueue } from '../../core/queue'; +import { getAllCodes, getAllUnits, firstSize } from '../../core/registry'; +import { resolvePrebidWindow, PrebidWindow } from '../../shared/globals'; type RequestBidsFunction = ( callbackOrOpts?: RequestAdsCallback | RequestAdsOptions, opts?: RequestAdsOptions diff --git a/crates/js/lib/src/ext/types.ts b/crates/js/lib/src/integrations/ext/types.ts similarity index 100% rename from crates/js/lib/src/ext/types.ts rename to crates/js/lib/src/integrations/ext/types.ts diff --git a/crates/js/lib/src/integrations/testlight.ts b/crates/js/lib/src/integrations/testlight/index.ts similarity index 90% rename from crates/js/lib/src/integrations/testlight.ts rename to crates/js/lib/src/integrations/testlight/index.ts index 6920b1e..3db00ce 100644 --- a/crates/js/lib/src/integrations/testlight.ts +++ b/crates/js/lib/src/integrations/testlight/index.ts @@ -1,7 +1,7 @@ -import type { TsjsApi } from '../core/types'; -import { installQueue } from '../core/queue'; -import { log } from '../core/log'; -import { resolvePrebidWindow, PrebidWindow } from '../shared/globals'; +import type { TsjsApi } from '../../core/types'; +import { installQueue } from '../../core/queue'; +import { log } from '../../core/log'; +import { resolvePrebidWindow, PrebidWindow } from '../../shared/globals'; type TestlightCallback = () => void; diff --git a/crates/js/lib/test/creative/click.test.ts b/crates/js/lib/test/integrations/creative/click.test.ts similarity index 100% rename from crates/js/lib/test/creative/click.test.ts rename to crates/js/lib/test/integrations/creative/click.test.ts diff --git a/crates/js/lib/test/creative/helpers.ts b/crates/js/lib/test/integrations/creative/helpers.ts similarity index 88% rename from crates/js/lib/test/creative/helpers.ts rename to crates/js/lib/test/integrations/creative/helpers.ts index 885d4e6..176a5e5 100644 --- a/crates/js/lib/test/creative/helpers.ts +++ b/crates/js/lib/test/integrations/creative/helpers.ts @@ -17,7 +17,7 @@ export const MUTATED_CLICK = 'https://example.com/landing?bar=2'; export const PROXY_RESPONSE = '/first-party/click?tsurl=https%3A%2F%2Fexample.com%2Flanding&bar=2&tstoken=newtoken'; -import type { TsCreativeConfig } from '../../src/shared/globals'; +import type { TsCreativeConfig } from '../../../src/shared/globals'; export async function importCreativeModule(config?: TsCreativeConfig): Promise { const globalRef = globalThis as { @@ -28,7 +28,7 @@ export async function importCreativeModule(config?: TsCreativeConfig): Promise = { - core: { - input: path.resolve(__dirname, 'src/core/index.ts'), - fileName: 'tsjs-core.js', - name: 'tsjs', - }, - ext: { - input: path.resolve(__dirname, 'src/ext/index.ts'), - fileName: 'tsjs-ext.js', - name: 'tsjs', - extend: true, - }, - creative: { - input: path.resolve(__dirname, 'src/creative/index.ts'), - fileName: 'tsjs-creative.js', - name: 'tscreative', - }, -}; + // Read TSJS_MODULES env var: creative,ext,testlight + // - If not set or empty string: include all integrations (default behavior) + // - If set to comma-separated list: include only those integrations + const modulesEnv = process.env.TSJS_MODULES; + const requestedModules = (modulesEnv || '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); -function discoverIntegrationBundles(): Record { - const integrationsDir = path.resolve(__dirname, 'src/integrations'); - if (!fs.existsSync(integrationsDir)) { - return {}; - } - const entries = fs.readdirSync(integrationsDir, { withFileTypes: true }); - return entries - .filter((entry) => entry.isFile() && entry.name.endsWith('.ts')) - .reduce>((acc, entry) => { - const slug = entry.name.replace(/\.ts$/i, ''); - const key = `integration:${slug}`; - acc[key] = { - input: path.resolve(integrationsDir, entry.name), - fileName: `tsjs-${slug}.js`, - name: `tsjs_${slug.replace(/[^a-zA-Z0-9]/g, '_')}`, - }; - return acc; - }, {}); -} + // Discover integration modules: directories in src/integrations/ with index.ts + const integrationModules = fs.existsSync(integrationsDir) + ? fs + .readdirSync(integrationsDir) + .filter((name) => { + const fullPath = path.join(integrationsDir, name); + const stat = fs.statSync(fullPath); + return ( + stat.isDirectory() && + fs.existsSync(path.join(fullPath, 'index.ts')) + ); + }) + : []; + + // Always include core first + const finalModules = ['core']; -const INTEGRATION_BUNDLES = discoverIntegrationBundles(); -const BUNDLES: Record = { ...BASE_BUNDLES, ...INTEGRATION_BUNDLES }; + // Add requested integrations based on TSJS_MODULES env var + if (requestedModules.length === 0) { + // TSJS_MODULES not set or empty: include all discovered integrations + finalModules.push(...integrationModules); + } else { + // TSJS_MODULES set to list: include only requested integrations (excluding 'core' as it's always added) + const requestedIntegrations = requestedModules.filter((m) => m !== 'core'); + finalModules.push( + ...integrationModules.filter((m) => requestedIntegrations.includes(m)) + ); + } -function resolveBundleKey(mode: string | undefined): string { - const fromEnv = process.env.TSJS_BUNDLE; - if (fromEnv && BUNDLES[fromEnv]) return fromEnv; + // Generate import statements + // Use namespace imports to capture all exports from each module + const importLines: string[] = []; + const exportEntries: string[] = []; - const normalized = mode?.toLowerCase(); - if (normalized && BUNDLES[normalized]) return normalized; + for (const moduleName of finalModules) { + if (moduleName === 'core') { + importLines.push(`import * as core from './core/index';`); + } else { + importLines.push( + `import * as ${moduleName} from './integrations/${moduleName}/index';` + ); + } + exportEntries.push(` ${moduleName},`); + } + + // Write the generated file + const output = `// Auto-generated by vite-plugin-tsjs-module-discovery +// DO NOT EDIT - This file is generated at build time based on TSJS_MODULES env var +${importLines.join('\n')} + +export const modules = { +${exportEntries.join('\n')} +}; - if (normalized) { - const integrationKey = `integration:${normalized}`; - if (BUNDLES[integrationKey]) { - return integrationKey; - } - } +export type ModuleName = ${finalModules.map((m) => `'${m}'`).join(' | ')}; +`; - return 'core'; + const generatedFilePath = path.join(srcDir, 'generated-modules.ts'); + fs.writeFileSync(generatedFilePath, output); + + console.log('[tsjs-module-discovery] Generated src/generated-modules.ts'); + console.log('[tsjs-module-discovery] Discovered integrations:', integrationModules); + console.log('[tsjs-module-discovery] Included modules:', finalModules); + }, + }; } -export default defineConfig(({ mode }) => { - const bundleKey = resolveBundleKey(mode); - const bundle = BUNDLES[bundleKey]; +export default defineConfig(() => { const distDir = path.resolve(__dirname, '../dist'); const buildTimestamp = new Date().toISOString(); const banner = `// build: ${buildTimestamp}\n`; @@ -85,32 +104,17 @@ export default defineConfig(({ mode }) => { sourcemap: false, minify: 'esbuild', rollupOptions: { - input: bundle.input, + input: path.resolve(__dirname, 'src/index.ts'), output: { format: 'iife', dir: distDir, - entryFileNames: bundle.fileName, + entryFileNames: 'tsjs-unified.js', inlineDynamicImports: true, - extend: bundle.extend ?? false, - name: bundle.name, + extend: false, + name: 'tsjs', }, }, }, - plugins: [createTimestampBannerPlugin(banner)], + plugins: [createModuleDiscoveryPlugin()], }; }); - -function createTimestampBannerPlugin(banner: string): Plugin { - return { - name: 'tsjs-build-timestamp-banner', - generateBundle(_options, bundleOutput) { - for (const file of Object.values(bundleOutput)) { - if (file.type === 'chunk') { - file.code = `${banner}${file.code}`; - } else if (file.type === 'asset' && typeof file.source === 'string') { - file.source = `${banner}${file.source}`; - } - } - }, - }; -} diff --git a/crates/js/src/bundle.rs b/crates/js/src/bundle.rs index ce3f1de..8b7e0ff 100644 --- a/crates/js/src/bundle.rs +++ b/crates/js/src/bundle.rs @@ -1,22 +1,80 @@ use hex::encode; use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use std::sync::OnceLock; -include!(concat!(env!("OUT_DIR"), "/bundle_manifest.rs")); +#[derive(Copy, Clone)] +struct TsjsMeta { + filename: &'static str, + bundle: &'static str, +} + +impl TsjsMeta { + const fn new(filename: &'static str, bundle: &'static str) -> Self { + Self { filename, bundle } + } +} + +const TSJS_BUNDLE_COUNT: usize = 1; + +#[repr(usize)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum TsjsBundle { + Unified, +} + +const METAS: [TsjsMeta; TSJS_BUNDLE_COUNT] = [ + TsjsMeta::new("tsjs-unified.js", include_str!(concat!(env!("OUT_DIR"), "/tsjs-unified.js"))), +]; + +const ALL_BUNDLES: [TsjsBundle; TSJS_BUNDLE_COUNT] = [ + TsjsBundle::Unified, +]; -static BUNDLE_MAP: OnceLock> = OnceLock::new(); +impl TsjsBundle { + pub const COUNT: usize = TSJS_BUNDLE_COUNT; + + pub const fn filename(self) -> &'static str { + METAS[self as usize].filename + } + + pub fn minified_filename(self) -> String { + let base = self.filename(); + match base.strip_suffix(".js") { + Some(stem) => format!("{stem}.min.js"), + None => format!("{base}.min.js"), + } + } + + pub(crate) const fn bundle(self) -> &'static str { + METAS[self as usize].bundle + } + + pub(crate) fn filename_map() -> &'static std::collections::HashMap<&'static str, TsjsBundle> { + static MAP: std::sync::OnceLock> = std::sync::OnceLock::new(); + + MAP.get_or_init(|| { + ALL_BUNDLES + .iter() + .copied() + .map(|bundle| (bundle.filename(), bundle)) + .collect::>() + }) + } + + pub fn from_filename(name: &str) -> Option { + Self::filename_map().get(name).copied() + } +} -fn bundles() -> &'static HashMap<&'static str, &'static str> { - BUNDLE_MAP.get_or_init(|| BUNDLES.iter().copied().collect()) +pub fn bundle_hash(bundle: TsjsBundle) -> String { + hash_bundle(bundle.bundle()) } pub fn bundle_for_filename(name: &str) -> Option<&'static str> { - bundles().get(name).copied() + TsjsBundle::from_filename(name).map(|bundle| bundle.bundle()) } -pub fn bundle_hash(filename: &str) -> Option { - bundle_for_filename(filename).map(hash_bundle) +pub fn bundle_hash_for_filename(name: &str) -> Option { + TsjsBundle::from_filename(name).map(|bundle| hash_bundle(bundle.bundle())) } fn hash_bundle(bundle: &'static str) -> String { diff --git a/crates/js/src/lib.rs b/crates/js/src/lib.rs index a4aa22d..7da69fc 100644 --- a/crates/js/src/lib.rs +++ b/crates/js/src/lib.rs @@ -1,3 +1,3 @@ pub mod bundle; -pub use bundle::{bundle_for_filename, bundle_hash}; +pub use bundle::{bundle_for_filename, bundle_hash, bundle_hash_for_filename, TsjsBundle}; From 787c642b69c511d5aa15335141f9664336d1f9bb Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Wed, 19 Nov 2025 18:27:53 -0800 Subject: [PATCH 7/9] Fixed formatting --- crates/js/build.rs | 8 +------- crates/js/src/bundle.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/crates/js/build.rs b/crates/js/build.rs index ba3fe57..da9fbe3 100644 --- a/crates/js/build.rs +++ b/crates/js/build.rs @@ -81,13 +81,7 @@ fn main() { copy_bundle(UNIFIED_BUNDLE, true, &crate_dir, &dist_dir, &out_dir); } -fn copy_bundle( - filename: &str, - required: bool, - crate_dir: &Path, - dist_dir: &Path, - out_dir: &Path, -) { +fn copy_bundle(filename: &str, required: bool, crate_dir: &Path, dist_dir: &Path, out_dir: &Path) { let primary = dist_dir.join(filename); let fallback = crate_dir.join("dist").join(filename); let target = out_dir.join(filename); diff --git a/crates/js/src/bundle.rs b/crates/js/src/bundle.rs index 8b7e0ff..ef29249 100644 --- a/crates/js/src/bundle.rs +++ b/crates/js/src/bundle.rs @@ -21,13 +21,12 @@ pub enum TsjsBundle { Unified, } -const METAS: [TsjsMeta; TSJS_BUNDLE_COUNT] = [ - TsjsMeta::new("tsjs-unified.js", include_str!(concat!(env!("OUT_DIR"), "/tsjs-unified.js"))), -]; +const METAS: [TsjsMeta; TSJS_BUNDLE_COUNT] = [TsjsMeta::new( + "tsjs-unified.js", + include_str!(concat!(env!("OUT_DIR"), "/tsjs-unified.js")), +)]; -const ALL_BUNDLES: [TsjsBundle; TSJS_BUNDLE_COUNT] = [ - TsjsBundle::Unified, -]; +const ALL_BUNDLES: [TsjsBundle; TSJS_BUNDLE_COUNT] = [TsjsBundle::Unified]; impl TsjsBundle { pub const COUNT: usize = TSJS_BUNDLE_COUNT; @@ -49,7 +48,8 @@ impl TsjsBundle { } pub(crate) fn filename_map() -> &'static std::collections::HashMap<&'static str, TsjsBundle> { - static MAP: std::sync::OnceLock> = std::sync::OnceLock::new(); + static MAP: std::sync::OnceLock> = + std::sync::OnceLock::new(); MAP.get_or_init(|| { ALL_BUNDLES From 4b9aaab640b0f45fcbb07eff23ce86e48ad131ea Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Wed, 19 Nov 2025 19:15:40 -0800 Subject: [PATCH 8/9] Added a way to remove scripts --- crates/common/src/html_processor.rs | 132 ++++++++++++++++-- crates/common/src/integrations/mod.rs | 7 +- crates/common/src/integrations/registry.rs | 102 ++++++++++++-- crates/common/src/integrations/testlight.rs | 31 ++-- crates/js/lib/src/index.ts | 7 - .../js/lib/src/integrations/ext/prebidjs.ts | 7 +- .../integrations/creative/proxy_sign.test.ts | 5 +- crates/js/lib/vite.config.ts | 23 +-- docs/integration_guide.md | 25 +++- 9 files changed, 268 insertions(+), 71 deletions(-) diff --git a/crates/common/src/html_processor.rs b/crates/common/src/html_processor.rs index e2f5f8e..d59ee8b 100644 --- a/crates/common/src/html_processor.rs +++ b/crates/common/src/html_processor.rs @@ -8,7 +8,8 @@ use lol_html::{element, html_content::ContentType, text, Settings as RewriterSet use regex::Regex; use crate::integrations::{ - IntegrationAttributeContext, IntegrationRegistry, IntegrationScriptContext, + AttributeRewriteOutcome, IntegrationAttributeContext, IntegrationRegistry, + IntegrationScriptContext, ScriptRewriteAction, }; use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; @@ -156,6 +157,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let original_href = href.clone(); if rewrite_prebid && is_prebid_script_url(&href) { el.remove(); + return Ok(()); } else { let new_href = href .replace(&patterns.https_origin(), &patterns.replacement_url()) @@ -165,7 +167,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } } - if let Some(integration_href) = integrations.rewrite_attribute( + match integrations.rewrite_attribute( "href", &href, &IntegrationAttributeContext { @@ -175,7 +177,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: &patterns.origin_host, }, ) { - href = integration_href; + AttributeRewriteOutcome::Unchanged => {} + AttributeRewriteOutcome::Replaced(integration_href) => { + href = integration_href; + } + AttributeRewriteOutcome::RemoveElement => { + el.remove(); + return Ok(()); + } } if href != original_href { @@ -195,6 +204,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let original_src = src.clone(); if rewrite_prebid && is_prebid_script_url(&src) { el.remove(); + return Ok(()); } else { let new_src = src .replace(&patterns.https_origin(), &patterns.replacement_url()) @@ -204,7 +214,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } } - if let Some(integration_src) = integrations.rewrite_attribute( + match integrations.rewrite_attribute( "src", &src, &IntegrationAttributeContext { @@ -214,7 +224,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: &patterns.origin_host, }, ) { - src = integration_src; + AttributeRewriteOutcome::Unchanged => {} + AttributeRewriteOutcome::Replaced(integration_src) => { + src = integration_src; + } + AttributeRewriteOutcome::RemoveElement => { + el.remove(); + return Ok(()); + } } if src != original_src { @@ -238,7 +255,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso action = new_action; } - if let Some(integration_action) = integrations.rewrite_attribute( + match integrations.rewrite_attribute( "action", &action, &IntegrationAttributeContext { @@ -248,7 +265,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: &patterns.origin_host, }, ) { - action = integration_action; + AttributeRewriteOutcome::Unchanged => {} + AttributeRewriteOutcome::Replaced(integration_action) => { + action = integration_action; + } + AttributeRewriteOutcome::RemoveElement => { + el.remove(); + return Ok(()); + } } if action != original_action { @@ -277,7 +301,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso srcset = new_srcset; } - if let Some(integration_srcset) = integrations.rewrite_attribute( + match integrations.rewrite_attribute( "srcset", &srcset, &IntegrationAttributeContext { @@ -287,7 +311,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: &patterns.origin_host, }, ) { - srcset = integration_srcset; + AttributeRewriteOutcome::Unchanged => {} + AttributeRewriteOutcome::Replaced(integration_srcset) => { + srcset = integration_srcset; + } + AttributeRewriteOutcome::RemoveElement => { + el.remove(); + return Ok(()); + } } if srcset != original_srcset { @@ -315,7 +346,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso imagesrcset = new_imagesrcset; } - if let Some(integration_imagesrcset) = integrations.rewrite_attribute( + match integrations.rewrite_attribute( "imagesrcset", &imagesrcset, &IntegrationAttributeContext { @@ -325,7 +356,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: &patterns.origin_host, }, ) { - imagesrcset = integration_imagesrcset; + AttributeRewriteOutcome::Unchanged => {} + AttributeRewriteOutcome::Replaced(integration_imagesrcset) => { + imagesrcset = integration_imagesrcset; + } + AttributeRewriteOutcome::RemoveElement => { + el.remove(); + return Ok(()); + } } if imagesrcset != original_imagesrcset { @@ -351,8 +389,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, }; - if let Some(rewritten) = rewriter.rewrite(text.as_str(), &ctx) { - text.replace(&rewritten, ContentType::Text); + match rewriter.rewrite(text.as_str(), &ctx) { + ScriptRewriteAction::Keep => {} + ScriptRewriteAction::Replace(rewritten) => { + text.replace(&rewritten, ContentType::Text); + } + ScriptRewriteAction::RemoveNode => { + text.remove(); + } } Ok(()) } @@ -399,8 +443,10 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso #[cfg(test)] mod tests { use super::*; + use crate::integrations::{AttributeRewriteAction, IntegrationAttributeRewriter}; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use std::io::Cursor; + use std::sync::Arc; fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { @@ -414,6 +460,66 @@ mod tests { } } + #[test] + fn integration_attribute_rewriter_can_remove_elements() { + struct RemovingLinkRewriter; + + impl IntegrationAttributeRewriter for RemovingLinkRewriter { + fn integration_id(&self) -> &'static str { + "removing" + } + + fn handles_attribute(&self, attribute: &str) -> bool { + attribute == "href" + } + + fn rewrite( + &self, + _attr_name: &str, + attr_value: &str, + _ctx: &IntegrationAttributeContext<'_>, + ) -> AttributeRewriteAction { + if attr_value.contains("remove-me") { + AttributeRewriteAction::remove_element() + } else { + AttributeRewriteAction::keep() + } + } + } + + let html = r#" + remove + keep + "#; + + let mut config = create_test_config(); + config.integrations = + IntegrationRegistry::from_rewriters(vec![Arc::new(RemovingLinkRewriter)], Vec::new()); + + let processor = create_html_processor(config); + let pipeline_config = PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }; + let mut pipeline = StreamingPipeline::new(pipeline_config, processor); + + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .unwrap(); + let processed = String::from_utf8(output).unwrap(); + + assert!( + processed.contains("keep-me"), + "Expected keep link to remain" + ); + assert!( + !processed.contains("remove-me"), + "Removing rewriter should drop matching elements" + ); + } + #[test] fn test_injects_unified_bundle_and_removes_prebid_refs() { let html = r#" diff --git a/crates/common/src/integrations/mod.rs b/crates/common/src/integrations/mod.rs index 190e160..e4577c4 100644 --- a/crates/common/src/integrations/mod.rs +++ b/crates/common/src/integrations/mod.rs @@ -6,9 +6,10 @@ mod registry; pub mod testlight; pub use registry::{ - IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, - IntegrationMetadata, IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, - IntegrationRegistry, IntegrationScriptContext, IntegrationScriptRewriter, + AttributeRewriteAction, AttributeRewriteOutcome, IntegrationAttributeContext, + IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationMetadata, IntegrationProxy, + IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, + IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, }; type IntegrationBuilder = fn(&Settings) -> Option; diff --git a/crates/common/src/integrations/registry.rs b/crates/common/src/integrations/registry.rs index f3ea428..b13e610 100644 --- a/crates/common/src/integrations/registry.rs +++ b/crates/common/src/integrations/registry.rs @@ -9,6 +9,67 @@ use fastly::{Request, Response}; use crate::error::TrustedServerError; use crate::settings::Settings; +/// Action returned by attribute rewriters to describe how the runtime should mutate the element. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttributeRewriteAction { + /// Leave the attribute and element untouched. + Keep, + /// Replace the attribute value with the provided string. + Replace(String), + /// Remove the entire element from the HTML stream. + RemoveElement, +} + +impl AttributeRewriteAction { + #[must_use] + pub fn keep() -> Self { + Self::Keep + } + + #[must_use] + pub fn replace(value: impl Into) -> Self { + Self::Replace(value.into()) + } + + #[must_use] + pub fn remove_element() -> Self { + Self::RemoveElement + } +} + +/// Outcome returned by the registry after running every matching attribute rewriter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttributeRewriteOutcome { + Unchanged, + Replaced(String), + RemoveElement, +} + +/// Action returned by inline script rewriters to describe how to mutate the node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScriptRewriteAction { + Keep, + Replace(String), + RemoveNode, +} + +impl ScriptRewriteAction { + #[must_use] + pub fn keep() -> Self { + Self::Keep + } + + #[must_use] + pub fn replace(value: impl Into) -> Self { + Self::Replace(value.into()) + } + + #[must_use] + pub fn remove_node() -> Self { + Self::RemoveNode + } +} + /// Context provided to integration HTML attribute rewriters. #[derive(Debug)] pub struct IntegrationAttributeContext<'a> { @@ -77,14 +138,14 @@ pub trait IntegrationAttributeRewriter: Send + Sync { fn integration_id(&self) -> &'static str; /// Return true when this rewriter wants to inspect a given attribute. fn handles_attribute(&self, attribute: &str) -> bool; - /// Attempt to rewrite the attribute value. Return `Some(new_value)` to - /// update the attribute or `None` to keep the original value. + /// Attempt to rewrite the attribute value. Return `AttributeRewriteAction::Replace` + /// to update the attribute, `Keep` to leave it untouched, or `RemoveElement` to drop the node. fn rewrite( &self, attr_name: &str, attr_value: &str, ctx: &IntegrationAttributeContext<'_>, - ) -> Option; + ) -> AttributeRewriteAction; } /// Trait for integration-provided inline script/text rewrite hooks. @@ -94,7 +155,7 @@ pub trait IntegrationScriptRewriter: Send + Sync { /// CSS selector (e.g. `script#__NEXT_DATA__`) that should trigger this rewriter. fn selector(&self) -> &'static str; /// Attempt to rewrite the inline text content for the selector. - fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> Option; + fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction; } /// Registration payload returned by integration builders. @@ -263,23 +324,29 @@ impl IntegrationRegistry { attr_name: &str, attr_value: &str, ctx: &IntegrationAttributeContext<'_>, - ) -> Option { + ) -> AttributeRewriteOutcome { let mut current = attr_value.to_string(); let mut changed = false; for rewriter in &self.inner.html_rewriters { if !rewriter.handles_attribute(attr_name) { continue; } - if let Some(next_value) = rewriter.rewrite(attr_name, ¤t, ctx) { - current = next_value; - changed = true; + match rewriter.rewrite(attr_name, ¤t, ctx) { + AttributeRewriteAction::Keep => {} + AttributeRewriteAction::Replace(next_value) => { + current = next_value; + changed = true; + } + AttributeRewriteAction::RemoveElement => { + return AttributeRewriteOutcome::RemoveElement; + } } } if changed { - Some(current) + AttributeRewriteOutcome::Replaced(current) } else { - None + AttributeRewriteOutcome::Unchanged } } @@ -317,4 +384,19 @@ impl IntegrationRegistry { map.into_values().collect() } + + #[cfg(test)] + pub fn from_rewriters( + attribute_rewriters: Vec>, + script_rewriters: Vec>, + ) -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner { + route_map: HashMap::new(), + routes: Vec::new(), + html_rewriters: attribute_rewriters, + script_rewriters, + }), + } + } } diff --git a/crates/common/src/integrations/testlight.rs b/crates/common/src/integrations/testlight.rs index a506ac6..419bd8f 100644 --- a/crates/common/src/integrations/testlight.rs +++ b/crates/common/src/integrations/testlight.rs @@ -12,8 +12,8 @@ use crate::backend::ensure_backend_from_url; use crate::constants::{HEADER_SYNTHETIC_FRESH, HEADER_SYNTHETIC_TRUSTED_SERVER}; use crate::error::TrustedServerError; use crate::integrations::{ - IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, - IntegrationProxy, IntegrationRegistration, + AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, + IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, }; use crate::settings::{IntegrationConfig as IntegrationConfigTrait, Settings}; use crate::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; @@ -193,20 +193,16 @@ impl IntegrationAttributeRewriter for TestlightIntegration { _attr_name: &str, attr_value: &str, _ctx: &IntegrationAttributeContext<'_>, - ) -> Option { + ) -> AttributeRewriteAction { if !self.config.rewrite_scripts { - return None; + return AttributeRewriteAction::keep(); } let lowered = attr_value.to_ascii_lowercase(); if lowered.contains("testlight.js") { - // TODO: need a way to remove the whole script tag - // None will still load external script Some will only rewrite attr - // but testlight script is now backed into the unified build - // for now this is loading the unified js again. - Some(self.config.shim_src.clone()) + AttributeRewriteAction::replace(self.config.shim_src.clone()) } else { - None + AttributeRewriteAction::keep() } } } @@ -277,9 +273,11 @@ mod tests { let rewritten = integration.rewrite("src", "https://cdn.testlight.net/v1/testlight.js", &ctx); - assert_eq!( - rewritten.as_deref(), - Some(shim_src.as_str()), + assert!( + matches!( + rewritten, + AttributeRewriteAction::Replace(ref value) if value == &shim_src + ), "Should swap integration script for trusted shim" ); } @@ -302,9 +300,10 @@ mod tests { origin_host: "origin.example.com", }; - assert!(integration - .rewrite("src", "https://cdn.testlight.net/script.js", &ctx) - .is_none()); + assert!(matches!( + integration.rewrite("src", "https://cdn.testlight.net/script.js", &ctx), + AttributeRewriteAction::Keep + )); } #[test] diff --git a/crates/js/lib/src/index.ts b/crates/js/lib/src/index.ts index 7cd2a5e..bfe6e69 100644 --- a/crates/js/lib/src/index.ts +++ b/crates/js/lib/src/index.ts @@ -1,17 +1,10 @@ // Unified tsjs bundle entry point // This file conditionally imports modules based on build-time configuration -import type { TsjsApi } from './core/types'; import { modules, type ModuleName } from './generated-modules'; import { log } from './core/log'; const VERSION = '0.1.0-unified'; -// Ensure we have a window object -const w: Window & { tsjs?: TsjsApi } = - ((globalThis as unknown as { window?: Window }).window as Window & { - tsjs?: TsjsApi; - }) || ({} as Window & { tsjs?: TsjsApi }); - // Log which modules are included in this build const includedModules = Object.keys(modules) as ModuleName[]; log.info('tsjs unified bundle initialized', { diff --git a/crates/js/lib/src/integrations/ext/prebidjs.ts b/crates/js/lib/src/integrations/ext/prebidjs.ts index 00837f7..242ebc0 100644 --- a/crates/js/lib/src/integrations/ext/prebidjs.ts +++ b/crates/js/lib/src/integrations/ext/prebidjs.ts @@ -1,5 +1,10 @@ // Prebid.js compatibility shim: exposes tsjs API through the legacy pbjs global. -import type { TsjsApi, HighestCpmBid, RequestAdsCallback, RequestAdsOptions } from '../../core/types'; +import type { + TsjsApi, + HighestCpmBid, + RequestAdsCallback, + RequestAdsOptions, +} from '../../core/types'; import { log } from '../../core/log'; import { installQueue } from '../../core/queue'; import { getAllCodes, getAllUnits, firstSize } from '../../core/registry'; diff --git a/crates/js/lib/test/integrations/creative/proxy_sign.test.ts b/crates/js/lib/test/integrations/creative/proxy_sign.test.ts index 58835de..41c86a8 100644 --- a/crates/js/lib/test/integrations/creative/proxy_sign.test.ts +++ b/crates/js/lib/test/integrations/creative/proxy_sign.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { shouldProxyExternalUrl, signProxyUrl } from '../../../src/integrations/creative/proxy_sign'; +import { + shouldProxyExternalUrl, + signProxyUrl, +} from '../../../src/integrations/creative/proxy_sign'; const ORIGINAL_FETCH = global.fetch; diff --git a/crates/js/lib/vite.config.ts b/crates/js/lib/vite.config.ts index 7b5ede2..8b9ed86 100644 --- a/crates/js/lib/vite.config.ts +++ b/crates/js/lib/vite.config.ts @@ -26,16 +26,11 @@ function createModuleDiscoveryPlugin(): Plugin { // Discover integration modules: directories in src/integrations/ with index.ts const integrationModules = fs.existsSync(integrationsDir) - ? fs - .readdirSync(integrationsDir) - .filter((name) => { - const fullPath = path.join(integrationsDir, name); - const stat = fs.statSync(fullPath); - return ( - stat.isDirectory() && - fs.existsSync(path.join(fullPath, 'index.ts')) - ); - }) + ? fs.readdirSync(integrationsDir).filter((name) => { + const fullPath = path.join(integrationsDir, name); + const stat = fs.statSync(fullPath); + return stat.isDirectory() && fs.existsSync(path.join(fullPath, 'index.ts')); + }) : []; // Always include core first @@ -48,9 +43,7 @@ function createModuleDiscoveryPlugin(): Plugin { } else { // TSJS_MODULES set to list: include only requested integrations (excluding 'core' as it's always added) const requestedIntegrations = requestedModules.filter((m) => m !== 'core'); - finalModules.push( - ...integrationModules.filter((m) => requestedIntegrations.includes(m)) - ); + finalModules.push(...integrationModules.filter((m) => requestedIntegrations.includes(m))); } // Generate import statements @@ -62,9 +55,7 @@ function createModuleDiscoveryPlugin(): Plugin { if (moduleName === 'core') { importLines.push(`import * as core from './core/index';`); } else { - importLines.push( - `import * as ${moduleName} from './integrations/${moduleName}/index';` - ); + importLines.push(`import * as ${moduleName} from './integrations/${moduleName}/index';`); } exportEntries.push(` ${moduleName},`); } diff --git a/docs/integration_guide.md b/docs/integration_guide.md index c364e6b..0f2fee6 100644 --- a/docs/integration_guide.md +++ b/docs/integration_guide.md @@ -140,6 +140,8 @@ headers. If the integration needs to rewrite script/link tags or inject HTML, implement `IntegrationAttributeRewriter` for attribute mutation and `IntegrationScriptRewriter` for inline `