diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 0cd2cd86a1..fb0bbe6ba1 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -58,6 +58,10 @@ steps: version=$(echo $full_version | cut -d'-' -f1) prerelease=$(echo $full_version | cut -d'-' -f2-) + # Application Insights connection string identifying best-effort + # telemetry as coming from Trident's own CI/CD pipeline builds + AZURE_MONITOR_CONNECTION_STRING="InstrumentationKey=e32fc20f-2cc6-4d86-9e12-ab5d24b366f7;IngestionEndpoint=https://eastus2-3.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus2.livediagnostics.monitor.azure.com/;ApplicationId=fb8e8afb-99bd-4143-ae31-22f9070005e2" + # Build RPMs and export only the artifact tarball (no image load/unpack). # CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN is populated by the CargoAuthenticate task. outdir="/tmp/_rpm_artifacts" @@ -80,6 +84,7 @@ steps: --build-arg RPM_PACKAGES="$RPM_PACKAGES" \ --build-arg RUST_PACKAGE="$RUST_PACKAGE" \ --build-arg RPM_DEST="$RPM_DEST" \ + --build-arg AZURE_MONITOR_CONNECTION_STRING="$AZURE_MONITOR_CONNECTION_STRING" \ --target artifact \ --output type=local,dest="$outdir" \ . diff --git a/crates/trident/build.rs b/crates/trident/build.rs index 3e838be702..2d92b7b418 100644 --- a/crates/trident/build.rs +++ b/crates/trident/build.rs @@ -1,4 +1,5 @@ fn main() -> Result<(), Box> { println!("cargo:rerun-if-env-changed=TRIDENT_VERSION"); + println!("cargo:rerun-if-env-changed=AZURE_MONITOR_CONNECTION_STRING"); Ok(()) -} +} diff --git a/crates/trident/src/agentconfig.rs b/crates/trident/src/agentconfig.rs index 4c37a330ec..440b6ca1e1 100644 --- a/crates/trident/src/agentconfig.rs +++ b/crates/trident/src/agentconfig.rs @@ -7,30 +7,65 @@ use trident_api::{ error::TridentError, }; +/// Whether Trident should attempt to send tracing data to Application +/// Insights (best-effort, and only when a connection string was compiled +/// into the binary -- see [`crate::AZURE_MONITOR_CONNECTION_STRING`]). +/// +/// Defaults to [`TelemetryPreference::OptOut`]: telemetry is disabled unless +/// a user has explicitly opted in via the Agent Configuration file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TelemetryPreference { + /// Telemetry is disabled. Trident will not send any tracing data off + /// the host. + #[default] + OptOut, + /// Telemetry is enabled, best-effort, provided a connection string was + /// compiled into this Trident binary. + OptIn, +} + pub struct AgentConfig { datastore: PathBuf, + telemetry: TelemetryPreference, } impl AgentConfig { /// Load the AgentConfig from the default configuration file. pub fn load() -> Result { + Self::load_from_path(AGENT_CONFIG_PATH) + } + + /// Load the AgentConfig from an arbitrary path. Split out from [`load`] + /// so the parsing logic can be unit tested without touching + /// [`AGENT_CONFIG_PATH`]. + fn load_from_path(path: &str) -> Result { let mut config = Self { datastore: TRIDENT_DATASTORE_PATH_DEFAULT.into(), + telemetry: TelemetryPreference::default(), }; - if let Ok(contents) = std::fs::read_to_string(AGENT_CONFIG_PATH) { + if let Ok(contents) = std::fs::read_to_string(path) { for line in contents.lines() { - if let Some(path) = line.strip_prefix("DatastorePath=") { - config.datastore = path.trim().into(); + if let Some(value) = line.strip_prefix("DatastorePath=") { + config.datastore = value.trim().into(); + } else if let Some(value) = line.strip_prefix("Telemetry=") { + config.telemetry = match value.trim().to_ascii_lowercase().as_str() { + "optin" => TelemetryPreference::OptIn, + "optout" => TelemetryPreference::OptOut, + other => { + debug!( + "Unrecognized Telemetry setting '{other}' in agent \ + configuration file, defaulting to OptOut" + ); + TelemetryPreference::OptOut + } + }; } } } else { // If the config file does not exist, we proceed with defaults. // Only log this at debug level to avoid alarming users unnecessarily. - debug!( - "Agent configuration file not found at {}, using defaults", - AGENT_CONFIG_PATH - ); + debug!("Agent configuration file not found at {path}, using defaults"); } Ok(config) @@ -40,4 +75,85 @@ impl AgentConfig { pub fn datastore_path(&self) -> &Path { &self.datastore } + + /// Whether telemetry (best-effort tracing to Application Insights) is + /// enabled per the agent configuration file. Defaults to `false` + /// (opt-out) when unset or unrecognized. + pub fn telemetry_enabled(&self) -> bool { + matches!(self.telemetry, TelemetryPreference::OptIn) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_defaults_when_file_missing() { + let config = AgentConfig::load_from_path("/nonexistent/path/for/trident-tests.conf") + .expect("load_from_path should not fail even if the file is missing"); + assert_eq!( + config.datastore_path(), + Path::new(TRIDENT_DATASTORE_PATH_DEFAULT) + ); + assert!( + !config.telemetry_enabled(), + "telemetry must default to OptOut" + ); + } + + #[test] + fn test_telemetry_optin() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trident.conf"); + std::fs::write(&path, "Telemetry=OptIn\n").unwrap(); + + let config = AgentConfig::load_from_path(path.to_str().unwrap()).unwrap(); + assert!(config.telemetry_enabled()); + } + + #[test] + fn test_telemetry_optout_explicit() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trident.conf"); + std::fs::write(&path, "Telemetry=OptOut\n").unwrap(); + + let config = AgentConfig::load_from_path(path.to_str().unwrap()).unwrap(); + assert!(!config.telemetry_enabled()); + } + + #[test] + fn test_telemetry_is_case_insensitive() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trident.conf"); + std::fs::write(&path, "Telemetry=OPTIN\n").unwrap(); + + let config = AgentConfig::load_from_path(path.to_str().unwrap()).unwrap(); + assert!(config.telemetry_enabled()); + } + + #[test] + fn test_telemetry_unrecognized_value_defaults_optout() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trident.conf"); + std::fs::write(&path, "Telemetry=maybe\n").unwrap(); + + let config = AgentConfig::load_from_path(path.to_str().unwrap()).unwrap(); + assert!(!config.telemetry_enabled()); + } + + #[test] + fn test_datastore_and_telemetry_together() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("trident.conf"); + std::fs::write( + &path, + "DatastorePath=/custom/path.sqlite\nTelemetry=OptIn\n", + ) + .unwrap(); + + let config = AgentConfig::load_from_path(path.to_str().unwrap()).unwrap(); + assert!(config.telemetry_enabled()); + assert_eq!(config.datastore_path(), Path::new("/custom/path.sqlite")); + } } diff --git a/crates/trident/src/lib.rs b/crates/trident/src/lib.rs index 7e72766329..448964bc74 100644 --- a/crates/trident/src/lib.rs +++ b/crates/trident/src/lib.rs @@ -56,8 +56,13 @@ pub use crate::{ }, grpc_client::client_main, logging::{ - background_log::BackgroundLog, background_uploader::BackgroundUploader, - logfwd::LogForwarder, logstream::Logstream, tracestream::TraceStream, + appinsights::AppInsightsSender, + background_log::BackgroundLog, + background_uploader::{BackgroundUploadHandle, BackgroundUploader}, + logfwd::LogForwarder, + logstream::Logstream, + operation_context::run_with_operation, + tracestream::TraceStream, }, orchestrate::OrchestratorConnection, reboot::request_reboot_with_wait, @@ -82,6 +87,15 @@ lazy_static::lazy_static! { .expect("Failed to parse TRIDENT_VERSION as semver::Version"); } +/// Azure Monitor / Application Insights connection string, compiled in at +/// build time via the `AZURE_MONITOR_CONNECTION_STRING` environment +/// variable. Empty when the variable was not provided at build time. +pub const AZURE_MONITOR_CONNECTION_STRING: &str = + match option_env!("AZURE_MONITOR_CONNECTION_STRING") { + Some(v) => v, + None => "", + }; + /// Trident binary path. const TRIDENT_BINARY_PATH: &str = "/usr/bin/trident"; diff --git a/crates/trident/src/logging/appinsights.rs b/crates/trident/src/logging/appinsights.rs new file mode 100644 index 0000000000..2f2b74f1ac --- /dev/null +++ b/crates/trident/src/logging/appinsights.rs @@ -0,0 +1,657 @@ +//! Best-effort tracing sink that forwards Trident's metric/span tracing +//! events to Azure Monitor / Application Insights. +//! +//! This intentionally does not depend on the OpenTelemetry SDK or an +//! Application Insights client crate. It follows the same minimal approach +//! as [`super::tracestream::TraceSender`]: parse the Application Insights +//! *connection string* (`InstrumentationKey=;IngestionEndpoint=;...`) +//! ourselves and build the raw Application Insights `EventData` envelope. +//! +//! Sending is delegated to the same [`super::background_uploader`] used by +//! [`super::logstream::Logstream`]: `send_event` only enqueues the envelope +//! and returns immediately, so tracing-layer callbacks (which run on +//! whichever thread emitted the event) are never blocked on network I/O. +//! The background uploader performs the actual `POST` to +//! `${ingestion_endpoint}/v2/track` with a short, bounded timeout on its own +//! dedicated thread. Failures (enqueue, network, non-2xx, etc.) are +//! logged and otherwise swallowed -- telemetry must never be able to affect +//! servicing outcomes. + +use std::{ + collections::BTreeMap, + sync::{Arc, RwLock}, + time::{Duration, Instant}, +}; + +use log::trace; +use serde_json::{json, Value}; +use tracing::{ + field::{Field, Visit}, + span, Event, Subscriber, +}; +use tracing_subscriber::{layer::Layer, registry::LookupSpan}; +use url::Url; + +use super::{ + background_uploader::BackgroundUploadHandle, operation_context, tracestream::PLATFORM_INFO, +}; +use crate::TRIDENT_VERSION; + +/// Default Application Insights ingestion endpoint, used when the connection +/// string does not specify one explicitly. +const DEFAULT_INGESTION_ENDPOINT: &str = "https://dc.services.visualstudio.com"; + +/// Per-request total timeout, enforced by the background uploader. Telemetry +/// must never meaningfully delay Trident's actual work. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); + +/// `Content-Type` for the Application Insights ingestion request. +const CONTENT_TYPE_JSON: &str = "application/json"; + +/// A parsed Application Insights connection string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ConnParts { + /// Ingestion endpoint, trailing slash stripped (e.g. + /// `https://region.in.applicationinsights.azure.com`). + pub ingestion_endpoint: String, + /// Instrumentation key. + pub instrumentation_key: String, +} + +impl ConnParts { + /// The `POST` target: `${ingestion_endpoint}/v2/track`. + fn track_url(&self) -> Option { + Url::parse(&format!( + "{}/v2/track", + self.ingestion_endpoint.trim_end_matches('/') + )) + .ok() + } +} + +/// Parse an Application Insights connection string of the form +/// `InstrumentationKey=;IngestionEndpoint=https://...;...`. Returns `None` +/// if the string is empty, unparsable, or missing an instrumentation key. +/// +/// If `IngestionEndpoint` is not given explicitly, the endpoint is derived +/// from the sovereign-cloud `EndpointSuffix`/`Location` fields when present +/// (e.g. `EndpointSuffix=applicationinsights.azure.cn;Location=chinaeast2` -> +/// `https://chinaeast2.in.applicationinsights.azure.cn`), matching the +/// Application Insights SDKs' documented connection-string format. Only if +/// none of `IngestionEndpoint`/`EndpointSuffix` are present does this fall +/// back to the public Application Insights endpoint -- a sovereign-cloud +/// string must never be silently redirected to the public endpoint. +pub(crate) fn parse_connection_string(s: &str) -> Option { + let mut instrumentation_key: Option = None; + let mut ingestion_endpoint: Option = None; + let mut endpoint_suffix: Option = None; + let mut location: Option = None; + + for part in s.split(';') { + let part = part.trim(); + if part.is_empty() { + continue; + } + let Some((key, value)) = part.split_once('=') else { + continue; + }; + match key.trim().to_ascii_lowercase().as_str() { + "instrumentationkey" => instrumentation_key = Some(value.trim().to_string()), + "ingestionendpoint" => { + ingestion_endpoint = Some(value.trim().trim_end_matches('/').to_string()) + } + "endpointsuffix" => endpoint_suffix = Some(value.trim().trim_matches('/').to_string()), + "location" => location = Some(value.trim().to_string()), + _ => {} + } + } + + let instrumentation_key = instrumentation_key.filter(|k| !k.is_empty())?; + + let ingestion_endpoint = match ingestion_endpoint.filter(|e| !e.is_empty()) { + Some(explicit) => explicit, + None => match endpoint_suffix.filter(|s| !s.is_empty()) { + // Sovereign-cloud form: derive the ingestion endpoint from + // EndpointSuffix (+ optional Location), rather than assuming the + // public endpoint. + Some(suffix) => match location.filter(|l| !l.is_empty()) { + Some(location) => format!("https://{location}.in.{suffix}"), + // No location: Application Insights uses the global `dc` + // prefix for this form, matching the public endpoint's own + // `dc.services.visualstudio.com` shape. + None => format!("https://dc.{suffix}"), + }, + None => DEFAULT_INGESTION_ENDPOINT.to_string(), + }, + }; + + Some(ConnParts { + ingestion_endpoint, + instrumentation_key, + }) +} + +/// A visitor that records the fields of a tracing event/span as a +/// `BTreeMap`, mirroring [`super::tracestream::TraceEntryVisitor`]. +#[derive(Default)] +struct FieldVisitor { + fields: BTreeMap, +} + +impl Visit for FieldVisitor { + fn record_i64(&mut self, field: &Field, value: i64) { + self.fields.insert(field.name().to_string(), json!(value)); + } + + fn record_f64(&mut self, field: &Field, value: f64) { + self.fields.insert(field.name().to_string(), json!(value)); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.fields.insert(field.name().to_string(), json!(value)); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.fields.insert(field.name().to_string(), json!(value)); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.fields.insert(field.name().to_string(), json!(value)); + } + + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_string(), json!(format!("{value:?}"))); + } +} + +/// Timestamp recorded when a span is entered, used to compute execution time +/// on exit. +struct SpanStart(Instant); + +/// Renders a JSON value as a string, since Application Insights `EventData` +/// properties are a `Map`. +fn stringify(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// A `tracing_subscriber::Layer` that forwards Trident's metric events and +/// instrumented spans to Application Insights, best-effort. +/// +/// Only constructed when a connection string was compiled into the binary +/// (see [`crate::AZURE_MONITOR_CONNECTION_STRING`]) *and* telemetry has been +/// enabled via the Agent Configuration file (see +/// [`crate::agentconfig::AgentConfig::telemetry_enabled`]); see +/// [`AppInsightsSender::from_connection_string`]. +pub struct AppInsightsSender { + instrumentation_key: String, + track_url: Url, + uploader: BackgroundUploadHandle, + /// The same persistent, per-host correlation ID handle used by + /// `TraceStream`/`TraceSender` (see `TraceStream::correlation_id_handle`), + /// so Application Insights events can be correlated back to a specific + /// host installation the same way tracestream metrics already are. + correlation_id: Arc>>, +} + +impl AppInsightsSender { + /// Build a sender from an Application Insights connection string. + /// Returns `None` if the string is empty, fails to parse, the + /// ingestion endpoint does not form a valid URL, or that URL's scheme + /// is not `https`. + /// + /// The events sent through this sender include host identifiers (see + /// [`super::tracestream::PLATFORM_INFO`]), so a non-HTTPS endpoint -- + /// e.g. from a build-time typo/misconfiguration -- is rejected rather + /// than silently sending opted-in host telemetry in cleartext. Azure + /// Monitor ingestion endpoints require HTTPS. + pub fn from_connection_string( + connection_string: &str, + uploader: BackgroundUploadHandle, + correlation_id: Arc>>, + ) -> Option { + let parts = parse_connection_string(connection_string)?; + match parts.track_url() { + Some(url) if url.scheme() == "https" => {} + _ => { + trace!( + "Application Insights ingestion endpoint '{}' is not HTTPS, disabling telemetry", + parts.ingestion_endpoint + ); + return None; + } + } + Self::from_parts(parts, uploader, correlation_id) + } + + fn from_parts( + parts: ConnParts, + uploader: BackgroundUploadHandle, + correlation_id: Arc>>, + ) -> Option { + let track_url = parts.track_url()?; + Some(Self { + instrumentation_key: parts.instrumentation_key, + track_url, + uploader, + correlation_id, + }) + } + + /// Build and enqueue an Application Insights `EventData` envelope for + /// the background uploader to send, best-effort. This only serializes + /// the envelope and hands it off to the uploader's channel, so it never + /// blocks on network I/O. A serialization failure or a closed uploader + /// is logged here at `trace` level; a later network error or non-2xx + /// response is logged by the background uploader itself (at `error` + /// level, same as any other background upload). Either way the failure + /// is otherwise ignored -- it can never affect servicing outcomes. + fn send_event(&self, name: &str, mut properties: BTreeMap) { + properties.insert("trident_version".to_string(), json!(TRIDENT_VERSION)); + for (key, value) in PLATFORM_INFO.iter() { + properties.insert(key.clone(), json!(stringify(value))); + } + if let Ok(correlation_id) = self.correlation_id.read() { + if let Some(correlation_id) = correlation_id.as_ref() { + properties + .entry("correlation_id".to_string()) + .or_insert_with(|| json!(correlation_id)); + } + } + if let Some((operation_id, command)) = operation_context::current() { + properties + .entry("operation_id".to_string()) + .or_insert_with(|| json!(operation_id)); + properties + .entry("command".to_string()) + .or_insert_with(|| json!(command)); + } + + let string_properties: BTreeMap = properties + .into_iter() + .map(|(key, value)| (key, stringify(&value))) + .collect(); + + let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let envelope = json!({ + "name": "Microsoft.ApplicationInsights.Event", + "time": now, + "iKey": self.instrumentation_key, + "tags": { "ai.internal.sdkVersion": format!("trident:{TRIDENT_VERSION}") }, + "data": { + "baseType": "EventData", + "baseData": { + "ver": 2, + "name": name, + "properties": string_properties, + } + } + }); + + let body = match serde_json::to_vec(&envelope) { + Ok(b) => b, + Err(e) => { + trace!("Failed to serialize Application Insights event: {e}"); + return; + } + }; + + if let Err(e) = self.uploader.upload( + &self.track_url, + body, + REQUEST_TIMEOUT, + Some(CONTENT_TYPE_JSON), + ) { + trace!("Failed to enqueue Application Insights event: {e}"); + } + } +} + +/// The `Layer` implementation mirrors +/// [`super::tracestream::TraceSender`]'s event/span handling, but renders an +/// Application Insights `EventData` envelope instead of Trident's own +/// metrics-file format. +impl Layer for AppInsightsSender +where + S: Subscriber + for<'span> LookupSpan<'span>, +{ + fn enabled( + &self, + metadata: &tracing::Metadata<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) -> bool { + metadata.level() <= &tracing::Level::INFO + } + + fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { + let mut visitor = FieldVisitor::default(); + event.record(&mut visitor); + + let Some(metric_name) = visitor + .fields + .get("metric_name") + .and_then(|v| v.as_str()) + .map(str::to_string) + else { + // Not a metric event (e.g. a plain log line); nothing to forward. + return; + }; + + let properties: BTreeMap = visitor + .fields + .into_iter() + .filter(|(key, _)| key != "metric_name") + .collect(); + + self.send_event(&metric_name, properties); + } + + fn on_new_span( + &self, + attrs: &span::Attributes<'_>, + id: &span::Id, + ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + if let Some(span) = ctx.span(id) { + let mut visitor = FieldVisitor::default(); + attrs.record(&mut visitor); + span.extensions_mut().insert(visitor); + } + } + + fn on_enter(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) { + if let Some(span) = ctx.span(id) { + span.extensions_mut().insert(SpanStart(Instant::now())); + } + } + + fn on_exit(&self, id: &span::Id, ctx: tracing_subscriber::layer::Context<'_, S>) { + let Some(span) = ctx.span(id) else { + return; + }; + let Some(SpanStart(start)) = span.extensions_mut().remove::() else { + return; + }; + let Some(mut visitor) = span.extensions_mut().remove::() else { + return; + }; + + visitor.fields.insert( + "execution_time".to_string(), + json!(start.elapsed().as_secs_f64()), + ); + + self.send_event(span.name(), visitor.fields); + } + + fn on_record( + &self, + id: &span::Id, + values: &span::Record<'_>, + ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + if let Some(span) = ctx.span(id) { + if let Some(visitor) = span.extensions_mut().get_mut::() { + values.record(visitor); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_connection_string_full() { + let parts = parse_connection_string( + "InstrumentationKey=abc123;IngestionEndpoint=https://region.example/;LiveEndpoint=https://live.example/", + ) + .expect("should parse"); + assert_eq!(parts.instrumentation_key, "abc123"); + assert_eq!(parts.ingestion_endpoint, "https://region.example"); + assert_eq!( + parts.track_url(), + Some(Url::parse("https://region.example/v2/track").unwrap()) + ); + } + + #[test] + fn test_parse_connection_string_missing_endpoint_uses_default() { + let parts = parse_connection_string("InstrumentationKey=abc123").expect("should parse"); + assert_eq!(parts.instrumentation_key, "abc123"); + assert_eq!(parts.ingestion_endpoint, DEFAULT_INGESTION_ENDPOINT); + assert_eq!( + parts.track_url(), + Some(Url::parse(&format!("{DEFAULT_INGESTION_ENDPOINT}/v2/track")).unwrap()) + ); + } + + #[test] + /// Sovereign-cloud connection strings that specify `EndpointSuffix` (and + /// optionally `Location`) instead of `IngestionEndpoint` must derive the + /// matching sovereign ingestion endpoint, not silently fall back to the + /// public one. + fn test_parse_connection_string_endpoint_suffix_with_location() { + let parts = parse_connection_string( + "InstrumentationKey=abc123;EndpointSuffix=applicationinsights.azure.cn;Location=chinaeast2", + ) + .expect("should parse"); + assert_eq!(parts.instrumentation_key, "abc123"); + assert_eq!( + parts.ingestion_endpoint, + "https://chinaeast2.in.applicationinsights.azure.cn" + ); + } + + #[test] + fn test_parse_connection_string_endpoint_suffix_without_location() { + let parts = parse_connection_string( + "InstrumentationKey=abc123;EndpointSuffix=applicationinsights.azure.cn", + ) + .expect("should parse"); + assert_eq!( + parts.ingestion_endpoint, + "https://dc.applicationinsights.azure.cn" + ); + } + + #[test] + fn test_parse_connection_string_explicit_ingestion_endpoint_wins_over_suffix() { + let parts = parse_connection_string( + "InstrumentationKey=abc123;IngestionEndpoint=https://region.example/;EndpointSuffix=applicationinsights.azure.cn", + ) + .expect("should parse"); + assert_eq!(parts.ingestion_endpoint, "https://region.example"); + } + + #[test] + fn test_parse_connection_string_missing_ikey_is_none() { + assert!(parse_connection_string("IngestionEndpoint=https://region.example/").is_none()); + assert!(parse_connection_string( + "InstrumentationKey=;IngestionEndpoint=https://region.example/" + ) + .is_none()); + } + + #[test] + fn test_parse_connection_string_empty_is_none() { + assert!(parse_connection_string("").is_none()); + } + + #[test] + fn test_from_connection_string_empty_is_none() { + assert!(AppInsightsSender::from_connection_string( + "", + BackgroundUploadHandle::new_mock(), + Arc::new(RwLock::new(None)), + ) + .is_none()); + } + + #[test] + fn test_from_connection_string_builds_sender() { + let sender = AppInsightsSender::from_connection_string( + "InstrumentationKey=k;IngestionEndpoint=https://region.example/", + BackgroundUploadHandle::new_mock(), + Arc::new(RwLock::new(None)), + ) + .expect("should build sender"); + assert_eq!(sender.instrumentation_key, "k"); + assert_eq!( + sender.track_url, + Url::parse("https://region.example/v2/track").unwrap() + ); + } + + #[test] + /// A non-HTTPS ingestion endpoint (e.g. from a build-time typo/ + /// misconfiguration) must be rejected rather than silently accepted -- + /// events sent through this sender include host identifiers. + fn test_from_connection_string_rejects_non_https_endpoint() { + assert!(AppInsightsSender::from_connection_string( + "InstrumentationKey=k;IngestionEndpoint=http://region.example/", + BackgroundUploadHandle::new_mock(), + Arc::new(RwLock::new(None)), + ) + .is_none()); + } + + #[test] + fn test_stringify() { + assert_eq!(stringify(&json!("hello")), "hello"); + assert_eq!(stringify(&json!(42)), "42"); + assert_eq!(stringify(&json!(true)), "true"); + } +} + +#[cfg(feature = "functional-test")] +#[cfg_attr(not(test), allow(unused_imports, dead_code))] +mod functional_test { + use super::*; + + use std::{ + io::{Read, Write}, + net::{TcpListener, TcpStream}, + sync::mpsc::channel, + }; + + use pytest_gen::functional_test; + use tracing_subscriber::{filter, layer::SubscriberExt}; + + /// Reads a full HTTP request off `stream`. A single `TcpStream::read` + /// call is not guaranteed to return the entire request (TCP is a byte + /// stream, not message-oriented) -- keep reading until the header + /// terminator has arrived and, per the declared `Content-Length`, the + /// full body has too. + fn read_full_http_request(stream: &mut TcpStream) -> String { + let mut buf: Vec = Vec::new(); + let mut chunk = [0u8; 4096]; + + loop { + match stream.read(&mut chunk) { + Ok(0) => break, // Peer closed the connection. + Ok(n) => buf.extend_from_slice(&chunk[..n]), + Err(_) => break, + } + + let Some(headers_end) = find_subslice(&buf, b"\r\n\r\n") else { + continue; // Headers not fully received yet. + }; + let headers = String::from_utf8_lossy(&buf[..headers_end]); + let content_length: usize = headers + .lines() + .find_map(|line| { + line.to_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().to_string()) + }) + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + + let body_end = headers_end + content_length; + if buf.len() >= body_end { + buf.truncate(body_end); + break; + } + } + + String::from_utf8_lossy(&buf).to_string() + } + + fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) + .map(|pos| pos + needle.len()) + } + + /// Spins up a local TCP listener standing in for the Application + /// Insights ingestion endpoint, and confirms the sender actually posts a + /// well-formed `EventData` envelope to it over the network. + #[functional_test] + fn test_app_insights_sender_posts_event() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + + // command_start (fired by run_with_operation) and test_metric below + // are each posted as their own request, so accept and collect every + // connection the listener sees rather than assuming exactly one. + let (tx, rx) = channel(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let received = read_full_http_request(&mut stream); + // Send a minimal response so the client's request completes + // cleanly instead of hitting a connection reset. + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + if tx.send(received).is_err() { + break; + } + } + }); + + let uploader = crate::BackgroundUploader::new().expect("should build uploader"); + let sender = AppInsightsSender::from_parts( + ConnParts { + ingestion_endpoint: format!("http://{addr}"), + instrumentation_key: "test-key".to_string(), + }, + uploader.get_handle().expect("uploader should be alive"), + Arc::new(RwLock::new(Some("test-correlation-id".to_string()))), + ) + .expect("should build sender") + .with_filter(filter::LevelFilter::INFO); + + let _guard = + tracing::subscriber::set_default(tracing_subscriber::Registry::default().with(sender)); + + // Wrapping in run_with_operation confirms operation_id/command also + // reach the outgoing properties, alongside correlation_id above. + operation_context::run_with_operation("test_command", || { + tracing::info!(metric_name = "test_metric", value = true); + }); + + // Collect both requests (command_start + test_metric); order between + // them is not guaranteed, so gather everything seen within the + // timeout and assert across the combined traffic. + let mut requests = Vec::new(); + while requests.len() < 2 { + match rx.recv_timeout(std::time::Duration::from_secs(5)) { + Ok(req) => requests.push(req), + Err(e) => panic!("did not receive the expected requests: {e:?}"), + } + } + let combined = requests.join("\n"); + + assert!(combined.contains("POST /v2/track")); + assert!(combined.contains("\"name\":\"test_metric\"")); + assert!(combined.contains("\"iKey\":\"test-key\"")); + assert!(combined.contains("\"correlation_id\":\"test-correlation-id\"")); + assert!(combined.contains("\"command\":\"test_command\"")); + assert!(combined.contains("\"operation_id\":")); + } +} diff --git a/crates/trident/src/logging/background_uploader.rs b/crates/trident/src/logging/background_uploader.rs index 9dd3914d7a..9e6acac92e 100644 --- a/crates/trident/src/logging/background_uploader.rs +++ b/crates/trident/src/logging/background_uploader.rs @@ -25,6 +25,8 @@ struct UploadData { url: Url, body: Vec, timeout: Duration, + /// Optional `Content-Type` header value to attach to the request. + content_type: Option<&'static str>, } /// A background uploader that sends log data to a remote server asynchronously. @@ -93,12 +95,21 @@ impl BackgroundUploader { continue; } - let result = HTTP_ASYNC_CLIENT + let mut request = HTTP_ASYNC_CLIENT .post(upload.url.clone()) .timeout(upload.timeout) - .body(upload.body) + .body(upload.body); + if let Some(content_type) = upload.content_type { + request = request.header(reqwest::header::CONTENT_TYPE, content_type); + } + // Treat non-2xx responses the same as a network-level failure: a + // consumer (e.g. AppInsightsSender) may document that rejected + // requests count as failures, so surface them here rather than + // silently treating any response as success. + let result = request .send() - .await; + .await + .and_then(|response| response.error_for_status()); if let Err(e) = result { error!("Background upload failed: {e}"); @@ -112,9 +123,6 @@ impl BackgroundUploader { } ); } - - // Note: we don't particularly care much for the status code since - // this is just a generic implementation. } debug!("Background uploader loop has exited"); @@ -147,6 +155,7 @@ impl BackgroundUploadHandle { url: &Url, body: impl Into>, timeout: Duration, + content_type: Option<&'static str>, ) -> Result<(), Error> { if let Some(sender) = self.sender.upgrade() { sender @@ -154,6 +163,7 @@ impl BackgroundUploadHandle { url: url.clone(), body: body.into(), timeout, + content_type, }) .context("Failed to send data to background uploader") } else { @@ -208,7 +218,7 @@ mod tests { let url = Url::parse("http://example.invalid/upload").unwrap(); // After shutdown, the weak sender can't be upgraded so upload should error. let err = handle - .upload(&url, b"hello".to_vec(), Duration::from_millis(50)) + .upload(&url, b"hello".to_vec(), Duration::from_millis(50), None) .unwrap_err(); assert!( err.to_string().contains("shut down"), @@ -236,7 +246,7 @@ mod tests { let url = Url::parse(&server.url()).unwrap().join("/upload").unwrap(); handle - .upload(&url, body.as_bytes().to_vec(), Duration::from_secs(2)) + .upload(&url, body.as_bytes().to_vec(), Duration::from_secs(2), None) .unwrap(); // Drop uploader first to ensure the background thread finishes processing all queued @@ -272,6 +282,7 @@ mod tests { url, body: body.as_bytes().to_vec(), timeout: Duration::from_secs(2), + content_type: None, }) .unwrap(); @@ -323,6 +334,7 @@ mod tests { url: Url::parse(&server.url()).unwrap().join("/slow").unwrap(), body: b"timeout-me".to_vec(), timeout: Duration::from_millis(100), + content_type: None, }) .unwrap(); @@ -332,6 +344,7 @@ mod tests { url: Url::parse(&server.url()).unwrap().join("/upload").unwrap(), body: b"this-should-be-skipped".to_vec(), timeout: Duration::from_secs(2), + content_type: None, }) .unwrap(); @@ -368,6 +381,7 @@ mod tests { url: Url::parse(&server.url()).unwrap().join("/queued").unwrap(), body: b"queued".to_vec(), timeout: Duration::from_secs(1), + content_type: None, }) .unwrap(); // Close the sender before running the loop to simulate shutdown. @@ -402,7 +416,7 @@ mod tests { let url = Url::parse(&server.url()).unwrap().join("/ok").unwrap(); handle - .upload(&url, b"hello".to_vec(), Duration::from_secs(2)) + .upload(&url, b"hello".to_vec(), Duration::from_secs(2), None) .unwrap(); // Drop the uploader to shut down the background thread. Both `handle` @@ -423,6 +437,7 @@ mod tests { &Url::parse(&server.url()).unwrap().join("/nope").unwrap(), b"nope".to_vec(), Duration::from_secs(1), + None, ) .unwrap_err(); assert!(err.to_string().contains("shut down")); diff --git a/crates/trident/src/logging/logstream.rs b/crates/trident/src/logging/logstream.rs index 8d19a36b2f..d46592f09e 100644 --- a/crates/trident/src/logging/logstream.rs +++ b/crates/trident/src/logging/logstream.rs @@ -177,7 +177,10 @@ impl Log for LogSender { // Send logs with a reasonably low timeout. The uploader will drop // logs if the server is unreachable or slow, or if it has been // closed. - if let Err(e) = self.uploader.upload(&target, body, Duration::from_secs(5)) { + if let Err(e) = self + .uploader + .upload(&target, body, Duration::from_secs(5), None) + { if !self.send_failed.swap(true, Ordering::Relaxed) { eprintln!("Failed to send log entry: {e}"); } diff --git a/crates/trident/src/logging/mod.rs b/crates/trident/src/logging/mod.rs index 082b1be2c2..a94293afdd 100644 --- a/crates/trident/src/logging/mod.rs +++ b/crates/trident/src/logging/mod.rs @@ -1,10 +1,12 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +pub(super) mod appinsights; pub(super) mod background_log; pub(super) mod background_uploader; pub(super) mod logfwd; pub(super) mod logstream; +pub(super) mod operation_context; pub(super) mod tracestream; #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/trident/src/logging/operation_context.rs b/crates/trident/src/logging/operation_context.rs new file mode 100644 index 0000000000..46eabb345a --- /dev/null +++ b/crates/trident/src/logging/operation_context.rs @@ -0,0 +1,116 @@ +//! Thread-local "which command is currently executing, and under what +//! operation ID" context, so telemetry sinks ([`super::tracestream::TraceSender`], +//! [`super::appinsights::AppInsightsSender`]) can tag every metric/span +//! fired during a command's execution with `command`/`operation_id` +//! fields, without every call site (deep in `engine::*`, `Trident::*`, +//! etc.) needing to pass them explicitly. +//! +//! A thread-local (rather than e.g. a `tracing` span) is enough here +//! because both places that set this context run the entire command +//! synchronously on a single, dedicated thread for the command's whole +//! duration: +//! - CLI: `run_trident`'s command dispatch (synchronous, main thread). +//! - gRPC/daemon: `servicing_request`'s closure runs inside +//! `tokio::task::spawn_blocking`, which gives it its own OS thread for +//! as long as the closure runs. +//! +//! `operation_id` is a fresh, random ID generated once per command +//! invocation (distinct from the persistent, per-host +//! `DataStore::correlation_id`, which is unrelated and set separately on +//! `TraceStream`/`AppInsightsSender`). + +use std::cell::RefCell; + +use uuid::Uuid; + +thread_local! { + static CURRENT_OPERATION: RefCell> = const { RefCell::new(None) }; +} + +/// Runs `f` with this thread tagged as executing `command`, under a fresh +/// `operation_id`. Also fires a `command_start` metric event immediately, +/// tagged the same way. Clears the tag afterwards (even if `f` panics, +/// via a drop guard), so a thread that runs multiple commands over its +/// lifetime (e.g. a thread pool worker reused across `spawn_blocking` +/// calls) never leaks a stale tag into an unrelated later command. +pub fn run_with_operation(command: &str, f: impl FnOnce() -> R) -> R { + let operation_id = Uuid::new_v4().to_string(); + + tracing::info!( + metric_name = "command_start", + command = command, + operation_id = operation_id.as_str(), + ); + + CURRENT_OPERATION.with(|cell| { + *cell.borrow_mut() = Some((operation_id, command.to_string())); + }); + + struct ClearOnDrop; + impl Drop for ClearOnDrop { + fn drop(&mut self) { + CURRENT_OPERATION.with(|cell| *cell.borrow_mut() = None); + } + } + let _clear = ClearOnDrop; + + f() +} + +/// Returns the `(operation_id, command)` pair set by +/// [`run_with_operation`] for the calling thread, if any. +pub(crate) fn current() -> Option<(String, String)> { + CURRENT_OPERATION.with(|cell| cell.borrow().clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_no_operation_by_default() { + assert!(current().is_none()); + } + + #[test] + fn test_run_with_operation_sets_and_clears_context() { + assert!(current().is_none()); + + let observed = run_with_operation("test_command", current); + let (operation_id, command) = observed.expect("context should be set inside f"); + assert_eq!(command, "test_command"); + assert_eq!(operation_id.len(), 36, "operation_id should be a UUID"); + + assert!( + current().is_none(), + "context must be cleared after run_with_operation returns" + ); + } + + #[test] + fn test_run_with_operation_clears_context_on_panic() { + assert!(current().is_none()); + + let result = std::panic::catch_unwind(|| { + run_with_operation("panicking_command", || { + panic!("boom"); + }) + }); + assert!(result.is_err()); + + assert!( + current().is_none(), + "context must be cleared even if f panics" + ); + } + + #[test] + fn test_each_invocation_gets_a_fresh_operation_id() { + let first = run_with_operation("cmd", || current().unwrap().0); + let second = run_with_operation("cmd", || current().unwrap().0); + assert_ne!( + first, second, + "each command invocation gets a fresh operation_id" + ); + } +} diff --git a/crates/trident/src/logging/tracestream.rs b/crates/trident/src/logging/tracestream.rs index c69bdeb8a6..db0ba1a8de 100644 --- a/crates/trident/src/logging/tracestream.rs +++ b/crates/trident/src/logging/tracestream.rs @@ -24,7 +24,7 @@ use osutils::{ uname, }; -use crate::{TRIDENT_METRICS_FILE_PATH, TRIDENT_VERSION}; +use crate::{logging::operation_context, TRIDENT_METRICS_FILE_PATH, TRIDENT_VERSION}; /// The product uuid is used to identify the hardware that Trident is running on. const PRODUCT_UUID_FILE: &str = "/sys/class/dmi/id/product_uuid"; @@ -140,6 +140,15 @@ impl TraceStream { } } + /// Returns a clone of the shared correlation-ID handle -- the same + /// underlying `Arc>` written by `set_correlation_id` -- so + /// other telemetry sinks (namely `AppInsightsSender`) can read the + /// current value at send-time without needing their own copy of the + /// logic that sets it. + pub fn correlation_id_handle(&self) -> Arc>> { + self.correlation_id.clone() + } + /// Create a Boxed TraceSender pub fn make_trace_sender(&self) -> Box { self.make_trace_sender_with_metrics_path(TRIDENT_METRICS_FILE_PATH) @@ -262,11 +271,12 @@ where }; // Apart from the metric name, check if we have a single or multiple values - let filtered_fields: BTreeMap = visitor + let mut filtered_fields: BTreeMap = visitor .fields .into_iter() .filter(|(key, _)| key != "metric_name") .collect(); + merge_operation_context(&mut filtered_fields); let value = if filtered_fields.len() > 1 { Value::Object(Map::from_iter(filtered_fields)) } else { @@ -356,6 +366,7 @@ where visitor .fields .insert("execution_time".to_string(), json!(execution_time)); + merge_operation_context(&mut visitor.fields); let entry = TraceEntry { timestamp: Utc::now(), @@ -401,6 +412,21 @@ where } } +/// Merge the current thread's `operation_id`/`command` (see +/// `operation_context`), if any, into `fields`. Values the caller already +/// set (e.g. an event that explicitly names its own `command`) are never +/// overwritten. +fn merge_operation_context(fields: &mut BTreeMap) { + if let Some((operation_id, command)) = operation_context::current() { + fields + .entry("operation_id".to_string()) + .or_insert_with(|| json!(operation_id)); + fields + .entry("command".to_string()) + .or_insert_with(|| json!(command)); + } +} + /// Obtain product uuid of the hardware Trident is running on fn read_product_uuid(filepath: String) -> String { match fs::read_to_string(filepath.clone()) { diff --git a/crates/trident/src/main.rs b/crates/trident/src/main.rs index bcd7268452..daa32c871f 100644 --- a/crates/trident/src/main.rs +++ b/crates/trident/src/main.rs @@ -2,7 +2,7 @@ use std::{fs, iter, panic, process::ExitCode}; use anyhow::{Context, Error}; use clap::Parser; -use log::{error, info, LevelFilter, Log}; +use log::{error, info, warn, LevelFilter, Log}; use osutils::logging::{filter::LogFilter, multilog::MultiLogger}; use trident::{ @@ -10,14 +10,29 @@ use trident::{ cli::{self, Cli, Commands, GetKind, TridentExitCodes}, init::offline, manual_rollback::{self, utils::ManualRollbackRequestKind}, - validation, BackgroundLog, BackgroundUploader, DataStore, ExitKind, LogForwarder, Logstream, - TraceStream, Trident, TRIDENT_BACKGROUND_LOG_PATH, + run_with_operation, validation, AppInsightsSender, BackgroundLog, BackgroundUploader, + DataStore, ExitKind, LogForwarder, Logstream, TraceStream, Trident, + TRIDENT_BACKGROUND_LOG_PATH, }; use trident_api::{ - config::HostConfigurationSource, + config::{HostConfigurationSource, Operations}, error::{InternalError, InvalidInputError, TridentError, TridentResultExt}, }; +/// Maps a base command name plus its requested `Operations` to the same +/// naming convention gRPC's `servicing_request` already uses for +/// stage/finalize granularity (e.g. `"install"` vs `"install_stage"` vs +/// `"install_finalize"`), so `command`/`operation_id` telemetry is +/// consistent regardless of whether the command came from the CLI or from +/// gRPC/daemon. +fn command_name(base: &str, ops: &Operations) -> String { + match (ops.has_stage(), ops.has_finalize()) { + (true, true) | (false, false) => base.to_string(), + (true, false) => format!("{base}_stage"), + (false, true) => format!("{base}_finalize"), + } +} + fn run_trident( mut logstream: Logstream, mut tracestream: TraceStream, @@ -178,39 +193,48 @@ fn run_trident( ref allowed_operations, multiboot, .. - } => trident - .install( - &mut datastore, - cli::to_operations(allowed_operations), - multiboot, - None, - ) - .map(|(exit_kind, _image_hash, _servicing_type)| exit_kind), + } => { + let ops = cli::to_operations(allowed_operations); + run_with_operation(&command_name("install", &ops), || { + trident + .install(&mut datastore, ops, multiboot, None) + .map(|(exit_kind, _image_hash, _servicing_type)| exit_kind) + }) + } Commands::Update { ref allowed_operations, .. - } => trident - .update(&mut datastore, cli::to_operations(allowed_operations)) - .map(|(exit_kind, _image_hash, _servicing_type)| exit_kind), - Commands::Commit { .. } => trident - .commit(&mut datastore) - .map(|(exit_kind, _servicing_type)| exit_kind), + } => { + let ops = cli::to_operations(allowed_operations); + run_with_operation(&command_name("update", &ops), || { + trident + .update(&mut datastore, ops) + .map(|(exit_kind, _image_hash, _servicing_type)| exit_kind) + }) + } + Commands::Commit { .. } => run_with_operation("commit", || { + trident + .commit(&mut datastore) + .map(|(exit_kind, _servicing_type)| exit_kind) + }), Commands::Rollback { runtime, ab, ref allowed_operations, .. - } => trident - .rollback( - &mut datastore, - runtime, - ab, - cli::to_operations(allowed_operations), - ) - .map(|(exit_kind, _servicing_type)| exit_kind), - Commands::RebuildRaid { .. } => trident - .rebuild_raid(&mut datastore) - .map(|()| ExitKind::Done), + } => { + let ops = cli::to_operations(allowed_operations); + run_with_operation(&command_name("rollback", &ops), || { + trident + .rollback(&mut datastore, runtime, ab, ops) + .map(|(exit_kind, _servicing_type)| exit_kind) + }) + } + Commands::RebuildRaid { .. } => run_with_operation("rebuild_raid", || { + trident + .rebuild_raid(&mut datastore) + .map(|()| ExitKind::Done) + }), _ => Err(TridentError::internal("Invalid command")), }; @@ -305,10 +329,80 @@ fn setup_logging( Ok(logstream) } -fn setup_tracing(args: &Cli) -> Result { - use tracing_subscriber::{filter, layer::SubscriberExt, Layer}; +/// Whether the Application Insights tracing layer ended up active on this +/// invocation, and why not when it didn't. Computed by [`setup_tracing`] and +/// surfaced via [`TelemetryStatus::log`] once real logging is available, so +/// operators can tell -- from the logs alone, without reading source -- +/// whether telemetry should be expected to actually reach Application +/// Insights, rather than silently assuming it based on the `Telemetry=` +/// setting alone (a bad/unreachable connection string, for example, fails +/// silently otherwise). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TelemetryStatus { + /// Tracing/telemetry setup does not apply to this command at all (the + /// `_ => {}` arm in [`setup_tracing`]) -- not logged. + NotApplicable, + /// `Telemetry=OptOut` (the default): telemetry was never attempted. + OptedOut, + /// Opted in, but no usable `AZURE_MONITOR_CONNECTION_STRING` was + /// compiled into this binary at build time (missing, empty, or failed + /// to parse). + NoConnectionString, + /// Opted in with a connection string, but the dedicated telemetry + /// background uploader is unavailable (failed to start, or its handle + /// was already closed). + UploaderUnavailable, + /// Opted in, connection string valid, uploader available: telemetry is + /// actively being sent. + Enabled, +} + +impl TelemetryStatus { + /// Log this status through the real logging pipeline. Must only be + /// called after logging has been initialized (`setup_logging`) -- + /// calling it earlier would silently no-op, since the `log` facade + /// drops everything until a logger is registered. + fn log(self) { + match self { + TelemetryStatus::NotApplicable => {} + TelemetryStatus::OptedOut => { + info!( + "Telemetry: disabled (Telemetry=OptOut, the default, in agent configuration)" + ); + } + TelemetryStatus::NoConnectionString => { + info!( + "Telemetry: opted in, but no usable Application Insights connection string \ + was compiled into this binary -- telemetry is a no-op" + ); + } + TelemetryStatus::UploaderUnavailable => { + warn!( + "Telemetry: opted in with a valid connection string, but the telemetry \ + background uploader is unavailable -- telemetry is a no-op" + ); + } + TelemetryStatus::Enabled => { + info!("Telemetry: enabled, sending tracing data to Application Insights"); + } + } + } +} + +fn setup_tracing( + args: &Cli, + telemetry_enabled: bool, + // Dedicated to Application Insights telemetry -- deliberately *not* the + // same `BackgroundUploader` instance used for log forwarding (see + // `main`), so a slow-but-successful telemetry endpoint can never build a + // backlog that delays real log uploads. `None` if telemetry is disabled + // or its uploader failed to start; either way telemetry becomes a no-op. + telemetry_uploader: Option<&BackgroundUploader>, +) -> Result<(TraceStream, TelemetryStatus), Error> { + use tracing_subscriber::{filter, layer::SubscriberExt, Layer, Registry}; let tracestream = TraceStream::default(); + let mut telemetry_status = TelemetryStatus::NotApplicable; match &args.command { Commands::Commit { .. } @@ -318,36 +412,67 @@ fn setup_tracing(args: &Cli) -> Result { | Commands::RebuildRaid { .. } | Commands::Rollback { check: false, .. } | Commands::Update { .. } => { + let mut layers: Vec + Send + Sync>> = vec![Box::new( + tracestream + .make_trace_sender() + .with_filter(filter::LevelFilter::INFO), + )]; + // As functionality moves to the Daemon, move the journald layer to // only be enabled for the Daemon command. Until then, keep it enabled // for all commands to ensure we have tracing info in journald for all // commands. - let baseline_tracing = tracing_subscriber::Registry::default().with( - tracestream - .make_trace_sender() - .with_filter(filter::LevelFilter::INFO), - ); - if let Ok(journald_layer) = tracing_journald::layer() { - tracing::subscriber::set_global_default( - baseline_tracing.with( + match tracing_journald::layer() { + Ok(journald_layer) => { + layers.push(Box::new( journald_layer .with_syslog_identifier("trident-tracing".to_string()) .with_filter(filter::LevelFilter::INFO), - ), - ) - .context("Failed to set global default subscriber")?; - } else { - eprintln!("Failed to connect to journald, falling back to tracing without journald support"); - tracing::subscriber::set_global_default(baseline_tracing) - .context("Failed to set global default subscriber")?; + )); + } + Err(_) => { + eprintln!("Failed to connect to journald, falling back to tracing without journald support"); + } } + + // Best-effort Application Insights telemetry: only added when the + // user has opted in via the Agent Configuration file *and* a + // connection string was compiled into this binary at build time. + // Never fails startup: an empty/unparsable connection string just + // means telemetry stays a no-op. `telemetry_status` records which + // of these applied so the caller can log it once real logging is + // available (see `TelemetryStatus::log`). + telemetry_status = if !telemetry_enabled { + TelemetryStatus::OptedOut + } else { + // A missing/closed uploader (e.g. its background thread + // failed to start) just means telemetry stays a no-op; it + // must never block or fail the rest of tracing setup. + match telemetry_uploader.and_then(|u| u.get_handle()) { + Some(handle) => match AppInsightsSender::from_connection_string( + trident::AZURE_MONITOR_CONNECTION_STRING, + handle, + tracestream.correlation_id_handle(), + ) { + Some(sender) => { + layers.push(Box::new(sender.with_filter(filter::LevelFilter::INFO))); + TelemetryStatus::Enabled + } + None => TelemetryStatus::NoConnectionString, + }, + None => TelemetryStatus::UploaderUnavailable, + } + }; + + tracing::subscriber::set_global_default(Registry::default().with(layers)) + .context("Failed to set global default subscriber")?; } _ => { // no op } } - Ok(tracestream) + Ok((tracestream, telemetry_status)) } fn main() -> ExitCode { @@ -363,13 +488,41 @@ fn main() -> ExitCode { } }; + // Whether best-effort Application Insights telemetry is enabled. Loaded + // early (before logging/tracing is set up) since the decision feeds + // directly into setup_tracing(). AgentConfig::load() never actually + // errors today, but default to disabled (OptOut) defensively if that + // ever changes. + let telemetry_enabled = AgentConfig::load() + .map(|config| config.telemetry_enabled()) + .unwrap_or(false); + + // Application Insights telemetry gets its own dedicated uploader/queue, + // entirely separate from `bg_uploader` (which carries real log + // forwarding). Both uploaders drain their queue sequentially on a single + // background thread, so sharing one between telemetry and logs would let + // a slow-but-successful telemetry endpoint build a backlog that delays + // operational log uploads. Failure to start is not fatal: telemetry + // simply becomes a no-op, mirroring failure handling on the handle + // itself. + let telemetry_uploader = telemetry_enabled + .then(|| match BackgroundUploader::new() { + Ok(uploader) => Some(uploader), + Err(e) => { + eprintln!("Failed to initialize telemetry uploader, disabling telemetry: {e:?}"); + None + } + }) + .flatten(); + // Initialize the telemetry flow - let tracestream = setup_tracing(&args); - if let Err(e) = tracestream { + let tracing_setup = setup_tracing(&args, telemetry_enabled, telemetry_uploader.as_ref()); + if let Err(e) = tracing_setup { // Defer to stderr since logging is not yet initialized. eprintln!("Failed to initialize tracing: {e:?}"); return TridentExitCodes::SetupFailed.into(); } + let (tracestream, telemetry_status) = tracing_setup.unwrap(); if let Commands::Daemon { inactivity_timeout, @@ -395,13 +548,14 @@ fn main() -> ExitCode { // Log version on startup info!("Trident version: {}", trident::TRIDENT_VERSION); + telemetry_status.log(); trident::server_main( log_forwarder, *inactivity_timeout, socket_path, logstream.unwrap(), - tracestream.unwrap(), + tracestream, ) } else if let Commands::GrpcClient(client_args) = &args.command { let logstream = setup_logging(&args, &bg_uploader, iter::empty()); @@ -414,6 +568,8 @@ fn main() -> ExitCode { error!("Failed to initialize logstream from environment: {e:?}"); } + telemetry_status.log(); + // Run the client command trident::client_main(client_args) } else { @@ -424,8 +580,10 @@ fn main() -> ExitCode { return TridentExitCodes::SetupFailed.into(); } + telemetry_status.log(); + // Invoke Trident - match run_trident(logstream.unwrap(), tracestream.unwrap(), &args) { + match run_trident(logstream.unwrap(), tracestream, &args) { Ok(ExitKind::Done) => {} Err(e) => { error!("{e:?}"); diff --git a/crates/trident/src/server/tridentserver/mod.rs b/crates/trident/src/server/tridentserver/mod.rs index 51434a3dbe..5a497c59d7 100644 --- a/crates/trident/src/server/tridentserver/mod.rs +++ b/crates/trident/src/server/tridentserver/mod.rs @@ -25,7 +25,7 @@ use trident_proto::v1::{ use crate::{ agentconfig::AgentConfig, - logging::logfwd::LogForwarder, + logging::{logfwd::LogForwarder, operation_context}, server::{activitytracker::ActivityTracker, support::stream::StreamWithLock}, ExitKind, Logstream, TraceStream, }; @@ -205,6 +205,15 @@ impl TridentServer { // Try to acquire the connection lock in write mode let guard = self.try_acquire_write_lock()?; + // Tag every metric/tracing event `f` fires (on whatever thread it + // ultimately runs on -- see `spawn_servicing_task`, which runs it + // via `tokio::task::spawn_blocking`, giving it a dedicated OS + // thread for its whole duration) with `command`/`operation_id`, the + // same way the CLI path does for its own dispatch. `name` already + // matches the CLI's own command-naming convention (see + // `command_name` in `main.rs`) for stage/finalize granularity. + let f = move || operation_context::run_with_operation(name, f); + // Create the gRPC response channel let (tx, rx) = mpsc::unbounded_channel(); diff --git a/crates/trident/src/subsystems/management.rs b/crates/trident/src/subsystems/management.rs index dbe408d2f7..1a6e3f39f0 100644 --- a/crates/trident/src/subsystems/management.rs +++ b/crates/trident/src/subsystems/management.rs @@ -96,24 +96,57 @@ fn configure_agent_config( if Path::new(agent_config_path).exists() { // If the agent config exists, check that the datastore matches the expected path. if let Ok(contents) = std::fs::read_to_string(agent_config_path) { + let mut datastore_path_line_present = false; let mut datastore_path_configured = TRIDENT_DATASTORE_PATH_DEFAULT; for line in contents.lines() { if let Some(path) = line.strip_prefix("DatastorePath=") { + datastore_path_line_present = true; datastore_path_configured = path.trim(); break; } } - // If the datastore path in the agent config does not match the expected path, - // return an error. - if datastore_path != Path::new(datastore_path_configured) { - return Err(TridentError::new( - InvalidInputError::ImageBadAgentConfiguration, - )) - .message(format!( - "Datastore path in agent config ({}) does not match expected path ({})", - datastore_path_configured, - datastore_path.display() - )); + + if datastore_path_line_present { + // An explicit DatastorePath= line is present: it must match + // the expected path exactly. + if datastore_path != Path::new(datastore_path_configured) { + return Err(TridentError::new( + InvalidInputError::ImageBadAgentConfiguration, + )) + .message(format!( + "Datastore path in agent config ({}) does not match expected path ({})", + datastore_path_configured, + datastore_path.display() + )); + } + } else if datastore_path != Path::new(TRIDENT_DATASTORE_PATH_DEFAULT) { + // No DatastorePath= line: the file may still carry other + // settings (e.g. Telemetry=) that must be preserved. Missing + // DatastorePath only implies the default path, so if a + // non-default path is expected, merge a DatastorePath= line + // into the existing file rather than treating it as a + // mismatch. + if is_root_verity { + // For root-verity, do not attempt to modify the agent config. + return Err(TridentError::new( + InvalidInputError::ImageBadAgentConfiguration, + )) + .message( + "Agent configuration file does not set a non-default datastore path \ + and root filesystem is verity", + ); + } + + let mut updated_contents = contents; + if !updated_contents.is_empty() && !updated_contents.ends_with('\n') { + updated_contents.push('\n'); + } + updated_contents.push_str(&format!("DatastorePath={}\n", datastore_path.display())); + fs::write(agent_config_path, updated_contents).structured( + ServicingError::CreateConfigurationFile { + path: agent_config_path.into(), + }, + )?; } } } else if datastore_path != Path::new(TRIDENT_DATASTORE_PATH_DEFAULT) { @@ -270,5 +303,60 @@ mod tests { ) .unwrap_err(); } + + { + // Regression test: agent config exists but only carries other + // settings (e.g. Telemetry=OptIn), with no DatastorePath= line. + // A non-default datastore path must be merged in, preserving the + // existing settings, rather than treated as a mismatch. + let agent_config_folder = tempfile::tempdir().unwrap(); + let agent_config_path = agent_config_folder.path().join("trident.conf"); + fs::write( + &agent_config_path, + "Telemetry=OptIn +", + ) + .unwrap(); + + configure_agent_config( + &agent_config_path.to_string_lossy(), + Path::new(nonstandard_datastore_path), + false, + ) + .unwrap(); + + let contents = std::fs::read_to_string(&agent_config_path).unwrap(); + assert!(contents.contains("Telemetry=OptIn")); + assert!(contents.contains(&format!("DatastorePath={nonstandard_datastore_path}"))); + } + + { + // Same as above, but root-verity: must not attempt to modify + // the agent config, and must error like the "file does not + // exist" root-verity case. + let agent_config_folder = tempfile::tempdir().unwrap(); + let agent_config_path = agent_config_folder.path().join("trident.conf"); + fs::write( + &agent_config_path, + "Telemetry=OptIn +", + ) + .unwrap(); + + configure_agent_config( + &agent_config_path.to_string_lossy(), + Path::new(nonstandard_datastore_path), + true, + ) + .unwrap_err(); + + // The file must be left untouched. + let contents = std::fs::read_to_string(&agent_config_path).unwrap(); + assert_eq!( + contents, + "Telemetry=OptIn +" + ); + } } } diff --git a/docs/Reference/Agent-Configuration.md b/docs/Reference/Agent-Configuration.md index 92c19c4c46..982ed24fe0 100644 --- a/docs/Reference/Agent-Configuration.md +++ b/docs/Reference/Agent-Configuration.md @@ -17,3 +17,56 @@ DatastorePath=/special/path/to/my-datastore.sqlite ``` > The datastore path cannot be hosted on an [A/B volume pair](./Glossary#ab-volume-pair) and must be an absolute path. + +## Telemetry + +Trident can optionally send a best-effort stream of its tracing data (the +same metrics/spans it already records locally to `/var/log/trident-metrics.jsonl`) +to Azure Monitor / Application Insights. This requires an Application +Insights connection string to have been compiled into the Trident binary at +build time (via the `AZURE_MONITOR_CONNECTION_STRING` environment variable); +if no connection string was compiled in, this setting has no effect. + +Every event sent also includes the following host metadata, so operators +should be aware this leaves the host along with the metrics/spans +themselves: + +- `asset_id`: the host's DMI product UUID (a stable hardware identifier). +- `os_release`: the `VERSION` field from `/etc/os-release`. +- `kernel_version`: the running kernel release (`uname -r`). +- `total_cpu`: the number of CPUs. +- `total_memory_gib`: total memory, in GiB. +- `trident_version`: the running Trident version. +- `correlation_id`: a random ID generated once and persisted in the + datastore, unique to this host installation. It is not derived from any + hardware/user identifier, but because it is stable across every Trident + invocation on this host, it does let separate events be correlated back + to the same installation over time. +- `operation_id`: a fresh, random ID generated for each individual command + invocation (e.g. one `trident update` run, or one gRPC request handled + by `tridentd`). Unlike `correlation_id`, this is never reused across + invocations -- it only lets events emitted *during the same command* be + correlated with each other. +- `command`: which command produced the event (e.g. `install`, `update`, + `update_stage`, `update_finalize`, `commit`, `rollback`, `rebuild_raid`). + The `_stage`/`_finalize` suffixes distinguish a two-step (stage-only or + finalize-only) invocation from a single combined one. + +Telemetry defaults to **disabled** (`OptOut`). To enable it, add a line to +the Agent Configuration file: + +``` conf +Telemetry=OptIn +``` + +The value is case-insensitive (`OptIn`, `optin`, and `OPTIN` are all +equivalent); any value other than a case-insensitive match for `OptIn` +(including an absent `Telemetry` line) is treated as `OptOut`. Telemetry +delivery is always best-effort and never +affects servicing outcomes, but failures are not all logged at the same +level: a failure to serialize an event, or to enqueue it because the +background uploader has already shut down, is logged at trace level, +while a failure to actually deliver an event (e.g. no network +connectivity, or a non-2xx response from Application Insights) is +logged at error level, so operators can find remote-delivery problems +in normal logs. diff --git a/packaging/docker/Dockerfile.full b/packaging/docker/Dockerfile.full index d2766bb73a..a4da4692be 100644 --- a/packaging/docker/Dockerfile.full +++ b/packaging/docker/Dockerfile.full @@ -36,6 +36,12 @@ ARG TRIDENT_VERSION=dev-build ARG RPM_VER=0.1.0 ARG RPM_REL=1 +# Application Insights connection string identifying telemetry as coming +# from Trident's own CI/CD pipeline builds. Set in pipeline template +# release.yml -- see trident.spec for how this is consumed +# (%{trident_azmon_conn_str}). +ARG AZURE_MONITOR_CONNECTION_STRING="" + ARG RPM_DEST=/usr/src/azl # This entry needs to exist in the config.toml file to allow cargo to use the @@ -52,7 +58,8 @@ RUN --mount=type=secret,id=registry_token \ rpmbuild -bb --build-in-place trident.spec \ --define="trident_version $TRIDENT_VERSION" \ --define="rpm_ver $RPM_VER" \ - --define="rpm_rel $RPM_REL" && \ + --define="rpm_rel $RPM_REL" \ + --define="trident_azmon_conn_str $AZURE_MONITOR_CONNECTION_STRING" && \ tar -czvf trident-rpms.tar.gz -C $RPM_DEST ./RPMS FROM scratch AS artifact diff --git a/packaging/rpm/trident.spec b/packaging/rpm/trident.spec index 9743a19879..de272f2fc3 100644 --- a/packaging/rpm/trident.spec +++ b/packaging/rpm/trident.spec @@ -11,6 +11,10 @@ %global selinuxtype targeted +# Azure Monitor / Application Insights connection string compiled into the +# azurelinux distro build of trident binary for best-effort telemetry. +%global trident_azmon_conn_str_public InstrumentationKey=cb38fc09-8473-4b4a-b5e4-208aa66a974f;IngestionEndpoint=https://eastus-8.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/;ApplicationId=b9814e3f-a121-4d99-9ba7-eeaf56195c29 + Summary: Declarative, security-first OS lifecycle agent designed primarily for Azure Linux Name: trident # Use hard-coded versions for distro build @@ -273,9 +277,25 @@ EOF %if %{undefined rpm_ver} # Use %{version}-%{release} for TRIDENT_VERSION in distro build export TRIDENT_VERSION="%{version}-%{release}" +# Public-usage placeholder connection string (see comment near the top of +# this spec file). +export AZURE_MONITOR_CONNECTION_STRING="%{trident_azmon_conn_str_public}" %else # Use %{trident_version} for Trident repo build export TRIDENT_VERSION="%{trident_version}" +# Connection string identifying telemetry as coming from Trident's own +# CI/CD pipeline builds (as opposed to azurelinux distro-package installs, +# which use the different, hardcoded connection string above). Hardcoded in +# .pipelines/templates/stages/trident_rpms/release.yml and passed through +# to this spec as an rpmbuild --define, the same way %{trident_version} is. +# +# Use the optional-expansion form (`%{?...}`): repo-build paths that define +# rpm_ver but do not pass --define trident_azmon_conn_str (e.g. +# packaging/docker/Dockerfile.full.public) must fall back to an empty +# string, matching the documented no-telemetry default -- not the literal, +# undefined `%{trident_azmon_conn_str}` text RPM would otherwise leave in +# place. +export AZURE_MONITOR_CONNECTION_STRING="%{?trident_azmon_conn_str}" %endif cargo build --release -p trident -p trident-acl-agent diff --git a/tests/images/azl-installer/installer-iso.yaml b/tests/images/azl-installer/installer-iso.yaml index a5f63a0a17..805d9106d6 100644 --- a/tests/images/azl-installer/installer-iso.yaml +++ b/tests/images/azl-installer/installer-iso.yaml @@ -72,6 +72,9 @@ os: - source: mos/scripts/installation.sh destination: /root/installer/installation.sh + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../common/trident.conf + destination: /etc/trident/trident.conf iso: initramfsType: full-os diff --git a/tests/images/azurelinux-direct-streaming-testimage/base/baseimg.yaml b/tests/images/azurelinux-direct-streaming-testimage/base/baseimg.yaml index 88a8d7b972..209e566c2b 100644 --- a/tests/images/azurelinux-direct-streaming-testimage/base/baseimg.yaml +++ b/tests/images/azurelinux-direct-streaming-testimage/base/baseimg.yaml @@ -78,3 +78,8 @@ os: services: enable: - sshd + + additionalFiles: + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf diff --git a/tests/images/common/trident.conf b/tests/images/common/trident.conf new file mode 100644 index 0000000000..a0ceefec56 --- /dev/null +++ b/tests/images/common/trident.conf @@ -0,0 +1 @@ +Telemetry=OptIn diff --git a/tests/images/trident-container-installer/base/baseimg.yaml b/tests/images/trident-container-installer/base/baseimg.yaml index 1e1dfded5a..fbcb20ee8a 100644 --- a/tests/images/trident-container-installer/base/baseimg.yaml +++ b/tests/images/trident-container-installer/base/baseimg.yaml @@ -57,6 +57,9 @@ os: - source: files/containerd-mmap-fix.cil destination: /usr/share/selinux/packages/containerd-mmap-fix.cil + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - trident-container diff --git a/tests/images/trident-container-testimage/base/baseimg.yaml b/tests/images/trident-container-testimage/base/baseimg.yaml index 53f3eff715..5a90a54f80 100644 --- a/tests/images/trident-container-testimage/base/baseimg.yaml +++ b/tests/images/trident-container-testimage/base/baseimg.yaml @@ -76,6 +76,9 @@ os: - source: files/containerd-mmap-fix.cil destination: /usr/share/selinux/packages/containerd-mmap-fix.cil + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - trident-container diff --git a/tests/images/trident-functest/base/baseimg.yaml b/tests/images/trident-functest/base/baseimg.yaml index 874463b5eb..d7ac3875c3 100644 --- a/tests/images/trident-functest/base/baseimg.yaml +++ b/tests/images/trident-functest/base/baseimg.yaml @@ -85,3 +85,8 @@ os: services: enable: - sshd + + additionalFiles: + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf diff --git a/tests/images/trident-installer/base/baseimg-direct-streaming.yaml b/tests/images/trident-installer/base/baseimg-direct-streaming.yaml index 3ace14fbda..26fa61cf47 100644 --- a/tests/images/trident-installer/base/baseimg-direct-streaming.yaml +++ b/tests/images/trident-installer/base/baseimg-direct-streaming.yaml @@ -87,6 +87,9 @@ os: - source: ../../../../tools/cmd/rcp-agent/rcp-agent.service destination: /usr/lib/systemd/system/rcp-agent.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf selinux: mode: enforcing diff --git a/tests/images/trident-installer/base/baseimg-split.yaml b/tests/images/trident-installer/base/baseimg-split.yaml index 6c5867d8a7..cd2032d72e 100644 --- a/tests/images/trident-installer/base/baseimg-split.yaml +++ b/tests/images/trident-installer/base/baseimg-split.yaml @@ -71,6 +71,9 @@ os: - source: files/trident-split-install.service destination: /usr/lib/systemd/system/trident-install.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf scripts: postCustomization: - path: post-install.sh diff --git a/tests/images/trident-installer/base/baseimg.yaml b/tests/images/trident-installer/base/baseimg.yaml index 7f6bf691bb..18a9d580a2 100644 --- a/tests/images/trident-installer/base/baseimg.yaml +++ b/tests/images/trident-installer/base/baseimg.yaml @@ -76,6 +76,9 @@ os: - source: files/trident-install.service destination: /usr/lib/systemd/system/trident-install.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf selinux: mode: enforcing diff --git a/tests/images/trident-mos/containerhost-iso.yaml b/tests/images/trident-mos/containerhost-iso.yaml index 586bb5ac9b..89f9248935 100644 --- a/tests/images/trident-mos/containerhost-iso.yaml +++ b/tests/images/trident-mos/containerhost-iso.yaml @@ -51,6 +51,9 @@ os: - source: files/download-trident-container.service destination: /usr/lib/systemd/system/download-trident-container.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - download-trident-container diff --git a/tests/images/trident-mos/iso.yaml b/tests/images/trident-mos/iso.yaml index 00b235a021..75ffd9acbe 100644 --- a/tests/images/trident-mos/iso.yaml +++ b/tests/images/trident-mos/iso.yaml @@ -94,6 +94,9 @@ os: - source: ../../../tools/cmd/rcp-agent/rcp-agent.service destination: /usr/lib/systemd/system/rcp-agent.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - rcp-agent diff --git a/tests/images/trident-testimage/base/baseimg.yaml b/tests/images/trident-testimage/base/baseimg.yaml index 80951ff4d6..9ef30e7b36 100644 --- a/tests/images/trident-testimage/base/baseimg.yaml +++ b/tests/images/trident-testimage/base/baseimg.yaml @@ -78,6 +78,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf scripts: postCustomization: - path: post-install.sh diff --git a/tests/images/trident-verity-testimage/base/baseimg-container.yaml b/tests/images/trident-verity-testimage/base/baseimg-container.yaml index 21dd824cb5..f3b2a75c6b 100644 --- a/tests/images/trident-verity-testimage/base/baseimg-container.yaml +++ b/tests/images/trident-verity-testimage/base/baseimg-container.yaml @@ -134,6 +134,9 @@ os: - source: files/trident-container.service destination: /usr/lib/systemd/system/trident-container.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-verity-testimage/base/baseimg.yaml b/tests/images/trident-verity-testimage/base/baseimg.yaml index 1477615dcd..85940ea7b5 100644 --- a/tests/images/trident-verity-testimage/base/baseimg.yaml +++ b/tests/images/trident-verity-testimage/base/baseimg.yaml @@ -132,6 +132,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-verity-testimage/usr/container.yaml b/tests/images/trident-verity-testimage/usr/container.yaml index 855cd0f10c..3a6a4fffc3 100644 --- a/tests/images/trident-verity-testimage/usr/container.yaml +++ b/tests/images/trident-verity-testimage/usr/container.yaml @@ -115,6 +115,9 @@ os: - source: files/trident-container.service destination: /usr/lib/systemd/system/trident-container.service + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf output: # Specifies config for output dir containining generated artifacts. This is # required for later running inject-files command, so that the final image diff --git a/tests/images/trident-verity-testimage/usr/host.yaml b/tests/images/trident-verity-testimage/usr/host.yaml index 476178d805..c4778f9bbe 100644 --- a/tests/images/trident-verity-testimage/usr/host.yaml +++ b/tests/images/trident-verity-testimage/usr/host.yaml @@ -121,6 +121,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf output: # Specifies config for output dir containining generated artifacts. This is # required for later running inject-files command, so that the final image diff --git a/tests/images/trident-vm-testimage/base/baseimg-grub-verity-azure.yaml b/tests/images/trident-vm-testimage/base/baseimg-grub-verity-azure.yaml index 7da14975f6..b9cf230ca1 100644 --- a/tests/images/trident-vm-testimage/base/baseimg-grub-verity-azure.yaml +++ b/tests/images/trident-vm-testimage/base/baseimg-grub-verity-azure.yaml @@ -145,6 +145,9 @@ os: - source: files/sudoers-wheel destination: /etc/sudoers.d/wheel + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-vm-testimage/base/baseimg-grub-verity.yaml b/tests/images/trident-vm-testimage/base/baseimg-grub-verity.yaml index 11715cfc91..f32930c357 100644 --- a/tests/images/trident-vm-testimage/base/baseimg-grub-verity.yaml +++ b/tests/images/trident-vm-testimage/base/baseimg-grub-verity.yaml @@ -143,6 +143,9 @@ os: - source: files/sudoers-wheel destination: /etc/sudoers.d/wheel + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-vm-testimage/base/baseimg-grub.yaml b/tests/images/trident-vm-testimage/base/baseimg-grub.yaml index 0286a9a8a5..569abd5391 100644 --- a/tests/images/trident-vm-testimage/base/baseimg-grub.yaml +++ b/tests/images/trident-vm-testimage/base/baseimg-grub.yaml @@ -84,6 +84,9 @@ os: - source: files/sudoers-wheel destination: /etc/sudoers.d/wheel + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf users: - name: testuser sshPublicKeyPaths: diff --git a/tests/images/trident-vm-testimage/base/baseimg-root-verity.yaml b/tests/images/trident-vm-testimage/base/baseimg-root-verity.yaml index 0ec362b70f..a6d14b3267 100644 --- a/tests/images/trident-vm-testimage/base/baseimg-root-verity.yaml +++ b/tests/images/trident-vm-testimage/base/baseimg-root-verity.yaml @@ -146,6 +146,9 @@ os: - source: files/sudoers-wheel destination: /etc/sudoers.d/wheel + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf overlays: - mountPoint: /etc lowerDirs: [/etc] diff --git a/tests/images/trident-vm-testimage/base/baseimg-usr-verity.yaml b/tests/images/trident-vm-testimage/base/baseimg-usr-verity.yaml index ae5208a2bb..86a5cbd796 100644 --- a/tests/images/trident-vm-testimage/base/baseimg-usr-verity.yaml +++ b/tests/images/trident-vm-testimage/base/baseimg-usr-verity.yaml @@ -145,6 +145,9 @@ os: - source: files/sudoers-wheel destination: /etc/sudoers.d/wheel + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - kdump diff --git a/tests/images/trident-vm-testimage/base/updateimg-grub-verity-azure.yaml b/tests/images/trident-vm-testimage/base/updateimg-grub-verity-azure.yaml index 6bde755308..587049fc33 100644 --- a/tests/images/trident-vm-testimage/base/updateimg-grub-verity-azure.yaml +++ b/tests/images/trident-vm-testimage/base/updateimg-grub-verity-azure.yaml @@ -136,6 +136,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-vm-testimage/base/updateimg-grub-verity.yaml b/tests/images/trident-vm-testimage/base/updateimg-grub-verity.yaml index db03d2c61f..4d251e9795 100644 --- a/tests/images/trident-vm-testimage/base/updateimg-grub-verity.yaml +++ b/tests/images/trident-vm-testimage/base/updateimg-grub-verity.yaml @@ -129,6 +129,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf services: enable: - etc-mount diff --git a/tests/images/trident-vm-testimage/base/updateimg-grub.yaml b/tests/images/trident-vm-testimage/base/updateimg-grub.yaml index 02b2cd0723..28a88c80b3 100644 --- a/tests/images/trident-vm-testimage/base/updateimg-grub.yaml +++ b/tests/images/trident-vm-testimage/base/updateimg-grub.yaml @@ -73,6 +73,9 @@ os: - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # Enable telemetry (Application Insights tracing) by default for test/dev images + - source: ../../common/trident.conf + destination: /etc/trident/trident.conf scripts: postCustomization: - path: scripts/post-install.sh