From 132060a4649c5b70a3b6c2f71f7f160c2c807bb8 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 4 Apr 2023 16:08:41 -0700 Subject: [PATCH 01/64] Serverless Trace Mini Agent (#124) --- Cargo.toml | 11 ++++++ src/lib.rs | 6 +++ src/mini_agent.rs | 87 ++++++++++++++++++++++++++++++++++++++++++ src/trace_flusher.rs | 55 ++++++++++++++++++++++++++ src/trace_processor.rs | 64 +++++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+) create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/mini_agent.rs create mode 100644 src/trace_flusher.rs create mode 100644 src/trace_processor.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..d3b8578 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "datadog-trace-mini-agent" +version = "0.1.0" +edition = "2021" + +[dependencies] +hyper = { version = "0.14", default-features = false, features = ["server"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"]} +async-trait = "0.1.64" +datadog-trace-protobuf = { path = "../trace-protobuf" } +datadog-trace-utils = { path = "../trace-utils" } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..372d42a --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,6 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +pub mod mini_agent; +pub mod trace_flusher; +pub mod trace_processor; diff --git a/src/mini_agent.rs b/src/mini_agent.rs new file mode 100644 index 0000000..10778f7 --- /dev/null +++ b/src/mini_agent.rs @@ -0,0 +1,87 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use datadog_trace_protobuf::pb; +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::sync::Arc; + +use tokio::sync::mpsc::{self, Receiver, Sender}; + +use crate::{trace_flusher, trace_processor}; + +const MINI_AGENT_PORT: usize = 8126; +const TRACE_ENDPOINT_PATH: &str = "/v0.4/traces"; +const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; + +pub struct MiniAgent { + pub trace_processor: Arc, + pub trace_flusher: Arc, +} + +impl MiniAgent { + #[tokio::main] + pub async fn start_mini_agent(&self) -> Result<(), Box> { + // setup a channel to send processed traces to our flusher + // tx is passed through each endpoint_handler to the trace processor, which uses it to send de-serialized processed + // trace payloads to our trace flusher. + let (tx, rx): (Sender, Receiver) = + mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); + + // start our trace flusher. receives trace payloads and handles buffering + deciding when to flush to backend. + let trace_flusher = self.trace_flusher.clone(); + tokio::spawn(async move { + let trace_flusher = trace_flusher.clone(); + trace_flusher.start_trace_flusher(rx).await; + }); + + // setup our hyper http server, where the endpoint_handler handles incoming requests + let trace_processor = self.trace_processor.clone(); + let make_svc = make_service_fn(move |_| { + let trace_processor = trace_processor.clone(); + let tx = tx.clone(); + + let service = service_fn(move |req| { + MiniAgent::trace_endpoint_handler(req, trace_processor.clone(), tx.clone()) + }); + + async move { Ok::<_, Infallible>(service) } + }); + + let addr = SocketAddr::from(([127, 0, 0, 1], MINI_AGENT_PORT as u16)); + let server = Server::bind(&addr).serve(make_svc); + + // start hyper http server + if let Err(e) = server.await { + println!("Server error: {}", e); + } + + Ok(()) + } + + async fn trace_endpoint_handler( + req: Request, + trace_processor: Arc, + tx: Sender, + ) -> Result, Infallible> { + match (req.method(), req.uri().path()) { + (&Method::PUT, TRACE_ENDPOINT_PATH) => { + match trace_processor.process_traces(req, tx).await { + Ok(res) => Ok(res), + Err(err) => Ok(Response::new(Body::from(format!( + "Error processing traces: {}", + err + )))), + } + } + _ => { + println!("{}", req.uri().path()); + let mut not_found = Response::default(); + *not_found.status_mut() = StatusCode::NOT_FOUND; + Ok(not_found) + } + } + } +} diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs new file mode 100644 index 0000000..e74e44b --- /dev/null +++ b/src/trace_flusher.rs @@ -0,0 +1,55 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use async_trait::async_trait; +use datadog_trace_protobuf::pb; + +use datadog_trace_utils::trace_utils; +use tokio::sync::mpsc::Receiver; + +const BUFFER_FLUSH_SIZE: usize = 1; + +#[async_trait] +pub trait TraceFlusher { + /// Starts a trace flusher that listens for trace payloads sent to the tokio mpsc Receiver, + /// implementing flushing logic that calls flush_traces. + async fn start_trace_flusher(&self, mut rx: Receiver); + /// Flushes traces to the Datadog trace intake. + async fn flush_traces(&self, traces: Vec); +} + +#[derive(Clone)] +pub struct ServerlessTraceFlusher {} + +#[async_trait] +impl TraceFlusher for ServerlessTraceFlusher { + async fn start_trace_flusher(&self, mut rx: Receiver) { + let mut buffer: Vec = Vec::with_capacity(BUFFER_FLUSH_SIZE); + + // receive trace payloads from http endpoint handlers and add them to the buffer. flush if + // the buffer gets to BUFFER_FLUSH_SIZE size. + while let Some(tracer_payload) = rx.recv().await { + buffer.push(tracer_payload); + if buffer.len() >= BUFFER_FLUSH_SIZE { + self.flush_traces(buffer.to_vec()).await; + buffer.clear(); + } + } + } + + async fn flush_traces(&self, traces: Vec) { + if traces.is_empty() { + return; + } + + let agent_payload = trace_utils::construct_agent_payload(traces); + let serialized_agent_payload = trace_utils::serialize_agent_payload(agent_payload); + + match trace_utils::send(serialized_agent_payload).await { + Ok(_) => {} + Err(e) => { + println!("Error sending trace: {:?}", e); + } + } + } +} diff --git a/src/trace_processor.rs b/src/trace_processor.rs new file mode 100644 index 0000000..4c4673e --- /dev/null +++ b/src/trace_processor.rs @@ -0,0 +1,64 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use async_trait::async_trait; +use datadog_trace_protobuf::pb; +use hyper::{http, Body, Request, Response}; + +use datadog_trace_utils::trace_utils; +use tokio::sync::mpsc::Sender; + +#[async_trait] +pub trait TraceProcessor { + /// Deserializes traces from a hyper request body and sends them through + /// the provided tokio mpsc Sender. + async fn process_traces( + &self, + req: Request, + tx: Sender, + ) -> http::Result>; +} + +#[derive(Clone)] +pub struct ServerlessTraceProcessor {} + +#[async_trait] +impl TraceProcessor for ServerlessTraceProcessor { + async fn process_traces( + &self, + req: Request, + tx: Sender, + ) -> http::Result> { + let (parts, body) = req.into_parts(); + let tracer_tags = trace_utils::get_tracer_tags_from_request_header(&parts.headers); + + // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf crate) + let traces = match trace_utils::get_traces_from_request_body(body).await { + Ok(res) => res, + Err(err) => { + return Response::builder().body(Body::from(format!( + "Error deserializing trace from request body: {}", + err + ))); + } + }; + + let trace_chunks: Vec = traces + .iter() + .map(|trace| trace_utils::construct_trace_chunk(trace.to_vec())) + .collect(); + + let tracer_payload = trace_utils::construct_tracer_payload(trace_chunks, tracer_tags); + + // send trace payload to our trace flusher + match tx.send(tracer_payload).await { + Ok(_) => Response::builder() + .status(200) + .body(Body::from("Successfully buffered traces to be flushed.")), + Err(e) => Response::builder().status(500).body(Body::from(format!( + "Error sending traces to the trace flusher. Error: {}", + e + ))), + } + } +} From 1c791bf857867392005d5b762e4b28be2bbddbd4 Mon Sep 17 00:00:00 2001 From: David Lee Date: Mon, 10 Apr 2023 13:10:45 -0700 Subject: [PATCH 02/64] Mini Agent: add trace normalization, root span calculation, logging (#130) --- Cargo.toml | 6 + src/mini_agent.rs | 15 ++- src/trace_flusher.rs | 12 +- src/trace_processor.rs | 260 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 269 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d3b8578..218f28c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,11 @@ edition = "2021" hyper = { version = "0.14", default-features = false, features = ["server"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" +log = "0.4" datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } +datadog-trace-normalization = { path = "../trace-normalization" } + +[dev-dependencies] +serde_json = "1.0" +rmp-serde = "1.1.1" \ No newline at end of file diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 10778f7..e30aee8 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -4,6 +4,7 @@ use datadog_trace_protobuf::pb; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use log::{error, info}; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; @@ -24,9 +25,8 @@ pub struct MiniAgent { impl MiniAgent { #[tokio::main] pub async fn start_mini_agent(&self) -> Result<(), Box> { - // setup a channel to send processed traces to our flusher - // tx is passed through each endpoint_handler to the trace processor, which uses it to send de-serialized processed - // trace payloads to our trace flusher. + // setup a channel to send processed traces to our flusher tx is passed through each endpoint_handler + // to the trace processor, which uses it to send de-serialized processed trace payloads to our trace flusher. let (tx, rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); @@ -51,11 +51,15 @@ impl MiniAgent { }); let addr = SocketAddr::from(([127, 0, 0, 1], MINI_AGENT_PORT as u16)); - let server = Server::bind(&addr).serve(make_svc); + let server_builder = Server::try_bind(&addr)?; + + let server = server_builder.serve(make_svc); + + info!("Mini Agent listening on port {}", MINI_AGENT_PORT); // start hyper http server if let Err(e) = server.await { - println!("Server error: {}", e); + error!("Server error: {}", e); } Ok(()) @@ -77,7 +81,6 @@ impl MiniAgent { } } _ => { - println!("{}", req.uri().path()); let mut not_found = Response::default(); *not_found.status_mut() = StatusCode::NOT_FOUND; Ok(not_found) diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index e74e44b..cfcc260 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -2,10 +2,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. use async_trait::async_trait; -use datadog_trace_protobuf::pb; +use log::error; +use tokio::sync::mpsc::Receiver; +use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; -use tokio::sync::mpsc::Receiver; const BUFFER_FLUSH_SIZE: usize = 1; @@ -45,11 +46,8 @@ impl TraceFlusher for ServerlessTraceFlusher { let agent_payload = trace_utils::construct_agent_payload(traces); let serialized_agent_payload = trace_utils::serialize_agent_payload(agent_payload); - match trace_utils::send(serialized_agent_payload).await { - Ok(_) => {} - Err(e) => { - println!("Error sending trace: {:?}", e); - } + if let Err(e) = trace_utils::send(serialized_agent_payload).await { + error!("Error sending trace: {:?}", e); } } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 4c4673e..6f377e7 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -4,14 +4,28 @@ use async_trait::async_trait; use datadog_trace_protobuf::pb; use hyper::{http, Body, Request, Response}; +use log::error; +use tokio::sync::mpsc::Sender; +use datadog_trace_normalization::normalizer; use datadog_trace_utils::trace_utils; -use tokio::sync::mpsc::Sender; + +macro_rules! parse_root_span_tags { + ( + $root_span_meta_map:ident, + { $($tag:literal => $($root_span_tags_struct_field:ident).+ ,)+ } + ) => { + $( + if let Some(root_span_tag_value) = $root_span_meta_map.get($tag) { + $($root_span_tags_struct_field).+ = root_span_tag_value; + } + )+ + } +} #[async_trait] pub trait TraceProcessor { - /// Deserializes traces from a hyper request body and sends them through - /// the provided tokio mpsc Sender. + /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc Sender. async fn process_traces( &self, req: Request, @@ -30,10 +44,11 @@ impl TraceProcessor for ServerlessTraceProcessor { tx: Sender, ) -> http::Result> { let (parts, body) = req.into_parts(); - let tracer_tags = trace_utils::get_tracer_tags_from_request_header(&parts.headers); + + let tracer_header_tags = trace_utils::get_tracer_header_tags(&parts.headers); // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf crate) - let traces = match trace_utils::get_traces_from_request_body(body).await { + let mut traces = match trace_utils::get_traces_from_request_body(body).await { Ok(res) => res, Err(err) => { return Response::builder().body(Body::from(format!( @@ -43,12 +58,56 @@ impl TraceProcessor for ServerlessTraceProcessor { } }; - let trace_chunks: Vec = traces - .iter() - .map(|trace| trace_utils::construct_trace_chunk(trace.to_vec())) - .collect(); + let mut trace_chunks: Vec = Vec::new(); - let tracer_payload = trace_utils::construct_tracer_payload(trace_chunks, tracer_tags); + let mut gathered_root_span_tags = false; + let mut root_span_tags = trace_utils::RootSpanTags::default(); + + for trace in traces.iter_mut() { + if let Err(e) = normalizer::normalize_trace(trace) { + error!("Error normalizing trace: {}", e); + } + + let mut chunk = trace_utils::construct_trace_chunk(trace.to_vec()); + + let root_span_index = match trace_utils::get_root_span_index(trace) { + Ok(res) => res, + Err(e) => { + error!( + "Error getting the root span index of a trace, skipping. {}", + e, + ); + continue; + } + }; + + if let Err(e) = normalizer::normalize_chunk(&mut chunk, root_span_index) { + error!("Error normalizing trace chunk: {}", e); + } + + if !tracer_header_tags.client_computed_top_level { + trace_utils::compute_top_level_span(&mut chunk.spans); + } + + trace_chunks.push(chunk); + + if !gathered_root_span_tags { + gathered_root_span_tags = true; + let meta_map = &trace[root_span_index].meta; + parse_root_span_tags!( + meta_map, + { + "env" => root_span_tags.env, + "version" => root_span_tags.app_version, + "_dd.hostname" => root_span_tags.hostname, + "runtime-id" => root_span_tags.runtime_id, + } + ); + } + } + + let tracer_payload = + trace_utils::construct_tracer_payload(trace_chunks, tracer_header_tags, root_span_tags); // send trace payload to our trace flusher match tx.send(tracer_payload).await { @@ -56,9 +115,188 @@ impl TraceProcessor for ServerlessTraceProcessor { .status(200) .body(Body::from("Successfully buffered traces to be flushed.")), Err(e) => Response::builder().status(500).body(Body::from(format!( - "Error sending traces to the trace flusher. Error: {}", + "Error sending traces to the trace flusher. {}", e ))), } } } + +#[cfg(test)] +mod tests { + use hyper::Request; + use serde_json::json; + use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, + }; + use tokio::sync::mpsc::{self, Receiver, Sender}; + + use crate::trace_processor::{self, TraceProcessor}; + use datadog_trace_protobuf::pb; + + fn create_test_span(start: i64, span_id: u64, parent_id: u64, is_top_level: bool) -> pb::Span { + let mut span = pb::Span { + trace_id: 111, + span_id, + service: "test-service".to_string(), + name: "test_name".to_string(), + resource: "test-resource".to_string(), + parent_id, + start, + duration: 5, + error: 0, + meta: HashMap::from([ + ("service".to_string(), "test-service".to_string()), + ("env".to_string(), "test-env".to_string()), + ( + "runtime-id".to_string(), + "afjksdljfkllksdj-28934889".to_string(), + ), + ]), + metrics: HashMap::new(), + r#type: "custom".to_string(), + meta_struct: HashMap::new(), + }; + if is_top_level { + span.metrics.insert("_top_level".to_string(), 1.0); + } + span + } + + fn create_test_json_span(start: i64, span_id: u64, parent_id: u64) -> serde_json::Value { + json!( + { + "trace_id": 111, + "span_id": span_id, + "service": "test-service", + "name": "test_name", + "resource": "test-resource", + "parent_id": parent_id, + "start": start, + "duration": 5, + "error": 0, + "meta": { + "service": "test-service", + "env": "test-env", + "runtime-id": "afjksdljfkllksdj-28934889", + }, + "metrics": {}, + "meta_struct": {}, + } + ) + } + + fn get_current_timestamp_nanos() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() as i64 + } + + #[tokio::test] + async fn test_process_trace() { + let (tx, mut rx): (Sender, Receiver) = + mpsc::channel(1); + + let start = get_current_timestamp_nanos(); + + let json_span = create_test_json_span(start, 222, 0); + + let bytes = rmp_serde::to_vec(&vec![vec![json_span]]).unwrap(); + let request = Request::builder() + .header("datadog-meta-tracer-version", "4.0.0") + .header("datadog-meta-lang", "nodejs") + .header("datadog-meta-lang-version", "v19.7.0") + .header("datadog-meta-lang-interpreter", "v8") + .header("datadog-container-id", "33") + .body(hyper::body::Body::from(bytes)) + .unwrap(); + + let trace_processor = trace_processor::ServerlessTraceProcessor {}; + let res = trace_processor.process_traces(request, tx).await; + assert!(res.is_ok()); + + let tracer_payload = rx.recv().await; + + assert!(tracer_payload.is_some()); + + let expected_tracer_payload = pb::TracerPayload { + container_id: "33".to_string(), + language_name: "nodejs".to_string(), + language_version: "v19.7.0".to_string(), + tracer_version: "4.0.0".to_string(), + runtime_id: "afjksdljfkllksdj-28934889".to_string(), + chunks: vec![pb::TraceChunk { + priority: 1, + origin: "".to_string(), + spans: vec![create_test_span(start, 222, 0, true)], + tags: HashMap::new(), + dropped_trace: false, + }], + tags: HashMap::new(), + env: "test-env".to_string(), + hostname: "".to_string(), + app_version: "".to_string(), + }; + + assert_eq!(expected_tracer_payload, tracer_payload.unwrap()); + } + + #[tokio::test] + async fn test_process_trace_top_level_span_set() { + let (tx, mut rx): (Sender, Receiver) = + mpsc::channel(1); + + let start = get_current_timestamp_nanos(); + + let json_trace = vec![ + create_test_json_span(start, 333, 222), + create_test_json_span(start, 222, 0), + create_test_json_span(start, 444, 333), + ]; + + let bytes = rmp_serde::to_vec(&vec![json_trace]).unwrap(); + let request = Request::builder() + .header("datadog-meta-tracer-version", "4.0.0") + .header("datadog-meta-lang", "nodejs") + .header("datadog-meta-lang-version", "v19.7.0") + .header("datadog-meta-lang-interpreter", "v8") + .header("datadog-container-id", "33") + .body(hyper::body::Body::from(bytes)) + .unwrap(); + + let trace_processor = trace_processor::ServerlessTraceProcessor {}; + let res = trace_processor.process_traces(request, tx).await; + assert!(res.is_ok()); + + let tracer_payload = rx.recv().await; + + assert!(tracer_payload.is_some()); + + let expected_tracer_payload = pb::TracerPayload { + container_id: "33".to_string(), + language_name: "nodejs".to_string(), + language_version: "v19.7.0".to_string(), + tracer_version: "4.0.0".to_string(), + runtime_id: "afjksdljfkllksdj-28934889".to_string(), + chunks: vec![pb::TraceChunk { + priority: 1, + origin: "".to_string(), + spans: vec![ + create_test_span(start, 333, 222, false), + create_test_span(start, 222, 0, true), + create_test_span(start, 444, 333, false), + ], + tags: HashMap::new(), + dropped_trace: false, + }], + tags: HashMap::new(), + env: "test-env".to_string(), + hostname: "".to_string(), + app_version: "".to_string(), + }; + + assert_eq!(expected_tracer_payload, tracer_payload.unwrap()); + } +} From b7f869410cd069ad078b61b6b06adabd703cc86d Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 25 Apr 2023 14:51:32 -0700 Subject: [PATCH 03/64] Mini Agent: Modify flushing logic, Serverless root span tags (#133) --- Cargo.toml | 2 +- src/mini_agent.rs | 50 ++++++++++++++++++++++++++++++-------- src/trace_flusher.rs | 48 ++++++++++++++++++++++++++---------- src/trace_processor.rs | 55 +++++++++++++++++++++++++++--------------- 4 files changed, 112 insertions(+), 43 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 218f28c..87bf4de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,10 @@ hyper = { version = "0.14", default-features = false, features = ["server"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" log = "0.4" +serde_json = "1.0" datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } datadog-trace-normalization = { path = "../trace-normalization" } [dev-dependencies] -serde_json = "1.0" rmp-serde = "1.1.1" \ No newline at end of file diff --git a/src/mini_agent.rs b/src/mini_agent.rs index e30aee8..76cf066 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -1,20 +1,21 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. -use datadog_trace_protobuf::pb; use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; use log::{error, info}; +use serde_json::json; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; - use tokio::sync::mpsc::{self, Receiver, Sender}; use crate::{trace_flusher, trace_processor}; +use datadog_trace_protobuf::pb; const MINI_AGENT_PORT: usize = 8126; const TRACE_ENDPOINT_PATH: &str = "/v0.4/traces"; +const INFO_ENDPOINT_PATH: &str = "/info"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; pub struct MiniAgent { @@ -55,11 +56,12 @@ impl MiniAgent { let server = server_builder.serve(make_svc); - info!("Mini Agent listening on port {}", MINI_AGENT_PORT); + info!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); // start hyper http server if let Err(e) = server.await { - error!("Server error: {}", e); + error!("Server error: {e}"); + return Err(e.into()); } Ok(()) @@ -69,17 +71,30 @@ impl MiniAgent { req: Request, trace_processor: Arc, tx: Sender, - ) -> Result, Infallible> { + ) -> http::Result> { match (req.method(), req.uri().path()) { (&Method::PUT, TRACE_ENDPOINT_PATH) => { match trace_processor.process_traces(req, tx).await { Ok(res) => Ok(res), - Err(err) => Ok(Response::new(Body::from(format!( - "Error processing traces: {}", - err - )))), + Err(err) => { + let error_message = format!("Error processing traces: {err}"); + let response_body = json!({ "message": error_message }).to_string(); + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(response_body)) + } } } + (_, INFO_ENDPOINT_PATH) => match Self::info_handler() { + Ok(res) => Ok(res), + Err(err) => { + let error_message = format!("Info endpoint error: {err}"); + let response_body = json!({ "message": error_message }).to_string(); + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(response_body)) + } + }, _ => { let mut not_found = Response::default(); *not_found.status_mut() = StatusCode::NOT_FOUND; @@ -87,4 +102,19 @@ impl MiniAgent { } } } + + fn info_handler() -> http::Result> { + let response_json = json!( + { + "endpoints": [ + TRACE_ENDPOINT_PATH, + INFO_ENDPOINT_PATH + ], + "client_drop_p0s": true, + } + ); + Response::builder() + .status(200) + .body(Body::from(response_json.to_string())) + } } diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index cfcc260..85e8e0b 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -2,14 +2,13 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. use async_trait::async_trait; -use log::error; -use tokio::sync::mpsc::Receiver; +use log::{debug, error, info}; +use std::{sync::Arc, time}; +use tokio::sync::{mpsc::Receiver, Mutex}; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; -const BUFFER_FLUSH_SIZE: usize = 1; - #[async_trait] pub trait TraceFlusher { /// Starts a trace flusher that listens for trace payloads sent to the tokio mpsc Receiver, @@ -25,13 +24,23 @@ pub struct ServerlessTraceFlusher {} #[async_trait] impl TraceFlusher for ServerlessTraceFlusher { async fn start_trace_flusher(&self, mut rx: Receiver) { - let mut buffer: Vec = Vec::with_capacity(BUFFER_FLUSH_SIZE); + let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let buffer_producer = buffer.clone(); + let buffer_consumer = buffer.clone(); + + tokio::spawn(async move { + while let Some(tracer_payload) = rx.recv().await { + let mut buffer = buffer_producer.lock().await; + buffer.push(tracer_payload); + } + }); - // receive trace payloads from http endpoint handlers and add them to the buffer. flush if - // the buffer gets to BUFFER_FLUSH_SIZE size. - while let Some(tracer_payload) = rx.recv().await { - buffer.push(tracer_payload); - if buffer.len() >= BUFFER_FLUSH_SIZE { + loop { + tokio::time::sleep(time::Duration::from_secs(3)).await; + + let mut buffer = buffer_consumer.lock().await; + if !buffer.is_empty() { self.flush_traces(buffer.to_vec()).await; buffer.clear(); } @@ -42,12 +51,25 @@ impl TraceFlusher for ServerlessTraceFlusher { if traces.is_empty() { return; } + info!("Flushing {} traces", traces.len()); + + debug!("Traces to be flushed: {traces:?}"); let agent_payload = trace_utils::construct_agent_payload(traces); - let serialized_agent_payload = trace_utils::serialize_agent_payload(agent_payload); + let serialized_agent_payload = match trace_utils::serialize_agent_payload(agent_payload) { + Ok(res) => res, + Err(err) => { + error!("Failed to serialize trace agent payload, dropping traces: {err}"); + return; + } + }; - if let Err(e) = trace_utils::send(serialized_agent_payload).await { - error!("Error sending trace: {:?}", e); + match trace_utils::send(serialized_agent_payload).await { + Ok(_) => info!("Successfully flushed traces"), + Err(e) => { + error!("Error sending trace: {:?}", e) + // TODO: Retries + } } } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 6f377e7..1299b0b 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -2,14 +2,16 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. use async_trait::async_trait; -use datadog_trace_protobuf::pb; use hyper::{http, Body, Request, Response}; -use log::error; +use log::{error, info}; +use serde_json::json; use tokio::sync::mpsc::Sender; use datadog_trace_normalization::normalizer; +use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; +/// Used to populate root_span_tags fields if they exist in the root span's meta tags macro_rules! parse_root_span_tags { ( $root_span_meta_map:ident, @@ -51,9 +53,9 @@ impl TraceProcessor for ServerlessTraceProcessor { let mut traces = match trace_utils::get_traces_from_request_body(body).await { Ok(res) => res, Err(err) => { + error!("Error deserializing trace from request body: err"); return Response::builder().body(Body::from(format!( - "Error deserializing trace from request body: {}", - err + "Error deserializing trace from request body: {err}" ))); } }; @@ -73,22 +75,28 @@ impl TraceProcessor for ServerlessTraceProcessor { let root_span_index = match trace_utils::get_root_span_index(trace) { Ok(res) => res, Err(e) => { - error!( - "Error getting the root span index of a trace, skipping. {}", - e, - ); + error!("Error getting the root span index of a trace, skipping. {e}"); continue; } }; if let Err(e) = normalizer::normalize_chunk(&mut chunk, root_span_index) { - error!("Error normalizing trace chunk: {}", e); + error!("Error normalizing trace chunk: {e}"); + } + + for span in chunk.spans.iter_mut() { + // TODO: obfuscate & truncate spans + if tracer_header_tags.client_computed_top_level { + trace_utils::update_tracer_top_level(span); + } } if !tracer_header_tags.client_computed_top_level { trace_utils::compute_top_level_span(&mut chunk.spans); } + trace_utils::set_serverless_root_span_tags(&mut chunk.spans[root_span_index]); + trace_chunks.push(chunk); if !gathered_root_span_tags { @@ -111,13 +119,18 @@ impl TraceProcessor for ServerlessTraceProcessor { // send trace payload to our trace flusher match tx.send(tracer_payload).await { - Ok(_) => Response::builder() - .status(200) - .body(Body::from("Successfully buffered traces to be flushed.")), - Err(e) => Response::builder().status(500).body(Body::from(format!( - "Error sending traces to the trace flusher. {}", - e - ))), + Ok(_) => { + info!("Successfully buffered traces to be flushed."); + let body = + json!({ "message": "Successfully buffered traces to be flushed." }).to_string(); + Response::builder().status(200).body(Body::from(body)) + } + Err(e) => { + let message = format!("Error sending traces to the trace flusher: {e}"); + error!("Error sending traces to the trace flusher: {message}"); + let body = json!({ "message": message }).to_string(); + Response::builder().status(500).body(Body::from(body)) + } } } } @@ -155,11 +168,15 @@ mod tests { ), ]), metrics: HashMap::new(), - r#type: "custom".to_string(), + r#type: "".to_string(), meta_struct: HashMap::new(), }; if is_top_level { span.metrics.insert("_top_level".to_string(), 1.0); + span.meta.insert("functionname".to_string(), "".to_string()); + span.meta + .insert("_dd.origin".to_string(), "gcp_function".to_string()); + span.r#type = "serverless".to_string(); } span } @@ -228,7 +245,7 @@ mod tests { tracer_version: "4.0.0".to_string(), runtime_id: "afjksdljfkllksdj-28934889".to_string(), chunks: vec![pb::TraceChunk { - priority: 1, + priority: i8::MIN as i32, origin: "".to_string(), spans: vec![create_test_span(start, 222, 0, true)], tags: HashMap::new(), @@ -281,7 +298,7 @@ mod tests { tracer_version: "4.0.0".to_string(), runtime_id: "afjksdljfkllksdj-28934889".to_string(), chunks: vec![pb::TraceChunk { - priority: 1, + priority: i8::MIN as i32, origin: "".to_string(), spans: vec![ create_test_span(start, 333, 222, false), From a5bafc5150ba66468cd1857a0474aaeb81dcf328 Mon Sep 17 00:00:00 2001 From: David Lee Date: Fri, 28 Apr 2023 12:25:35 -0700 Subject: [PATCH 04/64] Mini Agent: Trace stats (#139) --- src/config.rs | 38 ++++++++++ src/http_utils.rs | 166 +++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 + src/mini_agent.rs | 91 ++++++++++++++++------ src/stats_flusher.rs | 85 +++++++++++++++++++++ src/stats_processor.rs | 88 ++++++++++++++++++++++ src/trace_flusher.rs | 23 +++--- src/trace_processor.rs | 86 +++++++++++++++------ 8 files changed, 527 insertions(+), 54 deletions(-) create mode 100644 src/config.rs create mode 100644 src/http_utils.rs create mode 100644 src/stats_flusher.rs create mode 100644 src/stats_processor.rs diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..765f587 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use std::env; + +#[derive(Debug, Clone)] +pub struct Config { + pub api_key: String, + pub gcp_function_name: Option, + pub max_request_content_length: usize, + /// how often to flush traces, in seconds + pub trace_flush_interval: u64, + /// how often to flush stats, in seconds + pub stats_flush_interval: u64, +} + +impl Config { + pub fn new() -> Result> { + let api_key = env::var("DD_API_KEY")?; + let mut function_name = None; + + // Google cloud functions automatically sets either K_SERVICE or FUNCTION_NAME + // env vars to denote the cloud function name. + // K_SERVICE is set on newer runtimes, while FUNCTION_NAME is set on older deprecated runtimes. + if let Ok(res) = env::var("K_SERVICE") { + function_name = Some(res); + } else if let Ok(res) = env::var("FUNCTION_NAME") { + function_name = Some(res); + } + Ok(Config { + api_key, + gcp_function_name: function_name, + max_request_content_length: 10 * 1024 * 1024, // 10MB in Bytes + trace_flush_interval: 3, + stats_flush_interval: 3, + }) + } +} diff --git a/src/http_utils.rs b/src/http_utils.rs new file mode 100644 index 0000000..f826134 --- /dev/null +++ b/src/http_utils.rs @@ -0,0 +1,166 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use hyper::{ + header, + http::{self, HeaderMap}, + Body, Response, StatusCode, +}; +use log::{error, info}; +use serde_json::json; + +/// Does two things: +/// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be written, +/// otherwise error will be written. +/// 2. Returns the given message in the body of JSON response with the given status code. +/// +/// Response body format: +/// { +/// "message": message +/// } +pub fn log_and_create_http_response( + message: &str, + status: StatusCode, +) -> http::Result> { + if status.is_success() { + info!("{message}"); + } else { + error!("{message}"); + } + let body = json!({ "message": message }).to_string(); + Response::builder().status(status).body(Body::from(body)) +} + +/// Takes a request's header map, and verifies that the "content-length" header is present, valid, and less +/// than the given max_content_length. +/// +/// Will return None if no issues are found. Otherwise logs an error (with the given prefix) and returns +/// and HTTP Response with the appropriate error status code. +pub fn verify_request_content_length( + header_map: &HeaderMap, + max_content_length: usize, + error_message_prefix: &str, +) -> Option>> { + let content_length_header = match header_map.get(header::CONTENT_LENGTH) { + Some(res) => res, + None => { + return Some(log_and_create_http_response( + &format!("{error_message_prefix}: Missing Content-Length header"), + StatusCode::LENGTH_REQUIRED, + )); + } + }; + let header_as_string = match content_length_header.to_str() { + Ok(res) => res, + Err(_) => { + return Some(log_and_create_http_response( + &format!("{error_message_prefix}: Invalid Content-Length header"), + StatusCode::BAD_REQUEST, + )); + } + }; + let content_length = match header_as_string.to_string().parse::() { + Ok(res) => res, + Err(_) => { + return Some(log_and_create_http_response( + &format!("{error_message_prefix}: Invalid Content-Length header"), + StatusCode::BAD_REQUEST, + )); + } + }; + if content_length > max_content_length { + return Some(log_and_create_http_response( + &format!("{error_message_prefix}: Payload too large"), + StatusCode::PAYLOAD_TOO_LARGE, + )); + } + None +} + +#[cfg(test)] +mod tests { + use hyper::header; + use hyper::Body; + use hyper::HeaderMap; + use hyper::Response; + use hyper::StatusCode; + + use super::verify_request_content_length; + + fn create_test_headers_with_content_length(val: &str) -> HeaderMap { + let mut map = HeaderMap::new(); + map.insert(header::CONTENT_LENGTH, val.parse().unwrap()); + map + } + + async fn get_response_body_as_string(response: Response) -> String { + let body = response.into_body(); + let bytes = hyper::body::to_bytes(body).await.unwrap(); + String::from_utf8(bytes.into_iter().collect()).unwrap() + } + + #[tokio::test] + async fn test_request_content_length_missing() { + let verify_result = verify_request_content_length(&HeaderMap::new(), 1, "Test Prefix"); + assert!(verify_result.is_some()); + + let response = verify_result.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::LENGTH_REQUIRED); + assert_eq!( + get_response_body_as_string(response).await, + "{\"message\":\"Test Prefix: Missing Content-Length header\"}".to_string() + ); + } + + #[tokio::test] + async fn test_request_content_length_cant_convert_to_str() { + let verify_result = verify_request_content_length( + &create_test_headers_with_content_length("❤❤❤❤❤❤❤"), + 1, + "Test Prefix", + ); + assert!(verify_result.is_some()); + + let response = verify_result.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + get_response_body_as_string(response).await, + "{\"message\":\"Test Prefix: Invalid Content-Length header\"}".to_string() + ); + } + + #[tokio::test] + async fn test_request_content_length_cant_convert_to_usize() { + let verify_result = verify_request_content_length( + &create_test_headers_with_content_length("not_an_int"), + 1, + "Test Prefix", + ); + assert!(verify_result.is_some()); + + let response = verify_result.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + get_response_body_as_string(response).await, + "{\"message\":\"Test Prefix: Invalid Content-Length header\"}".to_string() + ); + } + + #[tokio::test] + async fn test_request_content_length_too_long() { + let verify_result = verify_request_content_length( + &create_test_headers_with_content_length("100"), + 1, + "Test Prefix", + ); + + assert!(verify_result.is_some()); + + let response = verify_result.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + get_response_body_as_string(response).await, + "{\"message\":\"Test Prefix: Payload too large\"}".to_string() + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 372d42a..ecec93a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,10 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +pub mod config; +pub mod http_utils; pub mod mini_agent; +pub mod stats_flusher; +pub mod stats_processor; pub mod trace_flusher; pub mod trace_processor; diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 76cf066..47f5f57 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -10,42 +10,82 @@ use std::net::SocketAddr; use std::sync::Arc; use tokio::sync::mpsc::{self, Receiver, Sender}; -use crate::{trace_flusher, trace_processor}; +use crate::http_utils::log_and_create_http_response; +use crate::{config, stats_flusher, stats_processor, trace_flusher, trace_processor}; use datadog_trace_protobuf::pb; const MINI_AGENT_PORT: usize = 8126; const TRACE_ENDPOINT_PATH: &str = "/v0.4/traces"; +const STATS_ENDPOINT_PATH: &str = "/v0.6/stats"; const INFO_ENDPOINT_PATH: &str = "/info"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; +const STATS_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; pub struct MiniAgent { + pub config: Arc, pub trace_processor: Arc, pub trace_flusher: Arc, + pub stats_processor: Arc, + pub stats_flusher: Arc, } impl MiniAgent { #[tokio::main] pub async fn start_mini_agent(&self) -> Result<(), Box> { - // setup a channel to send processed traces to our flusher tx is passed through each endpoint_handler + // setup a channel to send processed traces to our flusher. tx is passed through each endpoint_handler // to the trace processor, which uses it to send de-serialized processed trace payloads to our trace flusher. - let (tx, rx): (Sender, Receiver) = + let (trace_tx, trace_rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); // start our trace flusher. receives trace payloads and handles buffering + deciding when to flush to backend. let trace_flusher = self.trace_flusher.clone(); + let trace_config = self.config.clone(); tokio::spawn(async move { let trace_flusher = trace_flusher.clone(); - trace_flusher.start_trace_flusher(rx).await; + trace_flusher + .start_trace_flusher(trace_config.clone(), trace_rx) + .await; + }); + + // channels to send processed stats to our stats flusher. + let (stats_tx, stats_rx): ( + Sender, + Receiver, + ) = mpsc::channel(STATS_PAYLOAD_CHANNEL_BUFFER_SIZE); + + // start our stats flusher. + let stats_flusher = self.stats_flusher.clone(); + let stats_config = self.config.clone(); + tokio::spawn(async move { + let stats_flusher = stats_flusher.clone(); + stats_flusher + .start_stats_flusher(stats_config, stats_rx) + .await; }); // setup our hyper http server, where the endpoint_handler handles incoming requests let trace_processor = self.trace_processor.clone(); + let stats_processor = self.stats_processor.clone(); + let endpoint_config = self.config.clone(); + let make_svc = make_service_fn(move |_| { let trace_processor = trace_processor.clone(); - let tx = tx.clone(); + let trace_tx = trace_tx.clone(); + + let stats_processor = stats_processor.clone(); + let stats_tx = stats_tx.clone(); + + let endpoint_config = endpoint_config.clone(); let service = service_fn(move |req| { - MiniAgent::trace_endpoint_handler(req, trace_processor.clone(), tx.clone()) + MiniAgent::trace_endpoint_handler( + endpoint_config.clone(), + req, + trace_processor.clone(), + trace_tx.clone(), + stats_processor.clone(), + stats_tx.clone(), + ) }); async move { Ok::<_, Infallible>(service) } @@ -68,32 +108,38 @@ impl MiniAgent { } async fn trace_endpoint_handler( + config: Arc, req: Request, trace_processor: Arc, - tx: Sender, + trace_tx: Sender, + stats_processor: Arc, + stats_tx: Sender, ) -> http::Result> { match (req.method(), req.uri().path()) { (&Method::PUT, TRACE_ENDPOINT_PATH) => { - match trace_processor.process_traces(req, tx).await { + match trace_processor.process_traces(config, req, trace_tx).await { Ok(res) => Ok(res), - Err(err) => { - let error_message = format!("Error processing traces: {err}"); - let response_body = json!({ "message": error_message }).to_string(); - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(response_body)) - } + Err(err) => log_and_create_http_response( + &format!("Error processing traces: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } + (&Method::PUT, STATS_ENDPOINT_PATH) => { + match stats_processor.process_stats(config, req, stats_tx).await { + Ok(res) => Ok(res), + Err(err) => log_and_create_http_response( + &format!("Error processing trace stats: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), } } (_, INFO_ENDPOINT_PATH) => match Self::info_handler() { Ok(res) => Ok(res), - Err(err) => { - let error_message = format!("Info endpoint error: {err}"); - let response_body = json!({ "message": error_message }).to_string(); - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(response_body)) - } + Err(err) => log_and_create_http_response( + &format!("Info endpoint error: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), }, _ => { let mut not_found = Response::default(); @@ -108,6 +154,7 @@ impl MiniAgent { { "endpoints": [ TRACE_ENDPOINT_PATH, + STATS_ENDPOINT_PATH, INFO_ENDPOINT_PATH ], "client_drop_p0s": true, diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs new file mode 100644 index 0000000..20f60bc --- /dev/null +++ b/src/stats_flusher.rs @@ -0,0 +1,85 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use async_trait::async_trait; +use log::{debug, error, info}; +use std::{sync::Arc, time}; +use tokio::sync::{mpsc::Receiver, Mutex}; + +use datadog_trace_protobuf::pb; +use datadog_trace_utils::stats_utils; + +use crate::config::Config; + +#[async_trait] +pub trait StatsFlusher { + /// Starts a stats flusher that listens for stats payloads sent to the tokio mpsc Receiver, + /// implementing flushing logic that calls flush_stats. + async fn start_stats_flusher( + &self, + config: Arc, + mut rx: Receiver, + ); + /// Flushes stats to the Datadog trace stats intake. + async fn flush_stats(&self, config: Arc, traces: Vec); +} + +#[derive(Clone)] +pub struct ServerlessStatsFlusher {} + +#[async_trait] +impl StatsFlusher for ServerlessStatsFlusher { + async fn start_stats_flusher( + &self, + config: Arc, + mut rx: Receiver, + ) { + let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let buffer_producer = buffer.clone(); + let buffer_consumer = buffer.clone(); + + tokio::spawn(async move { + while let Some(stats_payload) = rx.recv().await { + let mut buffer = buffer_producer.lock().await; + buffer.push(stats_payload); + } + }); + + loop { + tokio::time::sleep(time::Duration::from_secs(config.stats_flush_interval)).await; + + let mut buffer = buffer_consumer.lock().await; + if !buffer.is_empty() { + self.flush_stats(config.clone(), buffer.to_vec()).await; + buffer.clear(); + } + } + } + + async fn flush_stats(&self, config: Arc, stats: Vec) { + if stats.is_empty() { + return; + } + info!("Flushing {} stats", stats.len()); + + let stats_payload = stats_utils::construct_stats_payload(stats); + + debug!("Stats payload to be sent: {stats_payload:?}"); + + let serialized_stats_payload = match stats_utils::serialize_stats_payload(stats_payload) { + Ok(res) => res, + Err(err) => { + error!("Failed to serialize stats payload, dropping stats: {err}"); + return; + } + }; + + match stats_utils::send_stats_payload(serialized_stats_payload, &config.api_key).await { + Ok(_) => info!("Successfully flushed stats"), + Err(e) => { + error!("Error sending stats: {e:?}") + } + } + } +} diff --git a/src/stats_processor.rs b/src/stats_processor.rs new file mode 100644 index 0000000..62d556d --- /dev/null +++ b/src/stats_processor.rs @@ -0,0 +1,88 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use hyper::{http, Body, Request, Response, StatusCode}; +use log::info; +use tokio::sync::mpsc::Sender; + +use datadog_trace_protobuf::pb; +use datadog_trace_utils::stats_utils; + +use crate::config::Config; +use crate::http_utils::{self, log_and_create_http_response}; + +#[async_trait] +pub trait StatsProcessor { + /// Deserializes trace stats from a hyper request body and sends them through + /// the provided tokio mpsc Sender. + async fn process_stats( + &self, + config: Arc, + req: Request, + tx: Sender, + ) -> http::Result>; +} + +#[derive(Clone)] +pub struct ServerlessStatsProcessor {} + +#[async_trait] +impl StatsProcessor for ServerlessStatsProcessor { + async fn process_stats( + &self, + config: Arc, + req: Request, + tx: Sender, + ) -> http::Result> { + let (parts, body) = req.into_parts(); + + if let Some(response) = http_utils::verify_request_content_length( + &parts.headers, + config.max_request_content_length, + "Error processing trace stats", + ) { + return response; + } + + info!("Recieved trace stats to process"); + + // deserialize trace stats from the request body, convert to protobuf structs (see trace-protobuf crate) + let mut stats: pb::ClientStatsPayload = + match stats_utils::get_stats_from_request_body(body).await { + Ok(res) => res, + Err(err) => { + return log_and_create_http_response( + &format!("Error deserializing trace stats from request body: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ); + } + }; + + let start = SystemTime::now(); + let timestamp = start + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + stats.stats[0].start = timestamp as u64; + + // send trace payload to our trace flusher + match tx.send(stats).await { + Ok(_) => { + return log_and_create_http_response( + "Successfully buffered stats to be flushed.", + StatusCode::ACCEPTED, + ); + } + Err(err) => { + return log_and_create_http_response( + &format!("Error sending stats to the stats flusher: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ); + } + } + } +} diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index 85e8e0b..ec4c06e 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -9,13 +9,15 @@ use tokio::sync::{mpsc::Receiver, Mutex}; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; +use crate::config::Config; + #[async_trait] pub trait TraceFlusher { /// Starts a trace flusher that listens for trace payloads sent to the tokio mpsc Receiver, /// implementing flushing logic that calls flush_traces. - async fn start_trace_flusher(&self, mut rx: Receiver); + async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver); /// Flushes traces to the Datadog trace intake. - async fn flush_traces(&self, traces: Vec); + async fn flush_traces(&self, config: Arc, traces: Vec); } #[derive(Clone)] @@ -23,7 +25,7 @@ pub struct ServerlessTraceFlusher {} #[async_trait] impl TraceFlusher for ServerlessTraceFlusher { - async fn start_trace_flusher(&self, mut rx: Receiver) { + async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver) { let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); let buffer_producer = buffer.clone(); @@ -37,25 +39,26 @@ impl TraceFlusher for ServerlessTraceFlusher { }); loop { - tokio::time::sleep(time::Duration::from_secs(3)).await; + tokio::time::sleep(time::Duration::from_secs(config.trace_flush_interval)).await; let mut buffer = buffer_consumer.lock().await; if !buffer.is_empty() { - self.flush_traces(buffer.to_vec()).await; + self.flush_traces(config.clone(), buffer.to_vec()).await; buffer.clear(); } } } - async fn flush_traces(&self, traces: Vec) { + async fn flush_traces(&self, config: Arc, traces: Vec) { if traces.is_empty() { return; } info!("Flushing {} traces", traces.len()); - debug!("Traces to be flushed: {traces:?}"); - let agent_payload = trace_utils::construct_agent_payload(traces); + + debug!("Trace agent payload to be sent: {agent_payload:?}"); + let serialized_agent_payload = match trace_utils::serialize_agent_payload(agent_payload) { Ok(res) => res, Err(err) => { @@ -64,10 +67,10 @@ impl TraceFlusher for ServerlessTraceFlusher { } }; - match trace_utils::send(serialized_agent_payload).await { + match trace_utils::send(serialized_agent_payload, &config.api_key).await { Ok(_) => info!("Successfully flushed traces"), Err(e) => { - error!("Error sending trace: {:?}", e) + error!("Error sending trace: {e:?}") // TODO: Retries } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 1299b0b..c775019 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -1,16 +1,22 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +use std::sync::Arc; + use async_trait::async_trait; -use hyper::{http, Body, Request, Response}; -use log::{error, info}; -use serde_json::json; +use hyper::{http, Body, Request, Response, StatusCode}; +use log::error; use tokio::sync::mpsc::Sender; use datadog_trace_normalization::normalizer; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; +use crate::{ + config::Config, + http_utils::{self, log_and_create_http_response}, +}; + /// Used to populate root_span_tags fields if they exist in the root span's meta tags macro_rules! parse_root_span_tags { ( @@ -30,6 +36,7 @@ pub trait TraceProcessor { /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc Sender. async fn process_traces( &self, + config: Arc, req: Request, tx: Sender, ) -> http::Result>; @@ -42,21 +49,30 @@ pub struct ServerlessTraceProcessor {} impl TraceProcessor for ServerlessTraceProcessor { async fn process_traces( &self, + config: Arc, req: Request, tx: Sender, ) -> http::Result> { let (parts, body) = req.into_parts(); + if let Some(response) = http_utils::verify_request_content_length( + &parts.headers, + config.max_request_content_length, + "Error processing traces", + ) { + return response; + } + let tracer_header_tags = trace_utils::get_tracer_header_tags(&parts.headers); // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf crate) let mut traces = match trace_utils::get_traces_from_request_body(body).await { Ok(res) => res, Err(err) => { - error!("Error deserializing trace from request body: err"); - return Response::builder().body(Body::from(format!( - "Error deserializing trace from request body: {err}" - ))); + return log_and_create_http_response( + &format!("Error deserializing trace from request body: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ); } }; @@ -67,7 +83,7 @@ impl TraceProcessor for ServerlessTraceProcessor { for trace in traces.iter_mut() { if let Err(e) = normalizer::normalize_trace(trace) { - error!("Error normalizing trace: {}", e); + error!("Error normalizing trace: {e}"); } let mut chunk = trace_utils::construct_trace_chunk(trace.to_vec()); @@ -95,7 +111,10 @@ impl TraceProcessor for ServerlessTraceProcessor { trace_utils::compute_top_level_span(&mut chunk.spans); } - trace_utils::set_serverless_root_span_tags(&mut chunk.spans[root_span_index]); + trace_utils::set_serverless_root_span_tags( + &mut chunk.spans[root_span_index], + config.gcp_function_name.clone(), + ); trace_chunks.push(chunk); @@ -120,16 +139,16 @@ impl TraceProcessor for ServerlessTraceProcessor { // send trace payload to our trace flusher match tx.send(tracer_payload).await { Ok(_) => { - info!("Successfully buffered traces to be flushed."); - let body = - json!({ "message": "Successfully buffered traces to be flushed." }).to_string(); - Response::builder().status(200).body(Body::from(body)) + return log_and_create_http_response( + "Successfully buffered traces to be flushed.", + StatusCode::ACCEPTED, + ); } - Err(e) => { - let message = format!("Error sending traces to the trace flusher: {e}"); - error!("Error sending traces to the trace flusher: {message}"); - let body = json!({ "message": message }).to_string(); - Response::builder().status(500).body(Body::from(body)) + Err(err) => { + return log_and_create_http_response( + &format!("Error sending traces to the trace flusher: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ); } } } @@ -141,11 +160,15 @@ mod tests { use serde_json::json; use std::{ collections::HashMap, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; use tokio::sync::mpsc::{self, Receiver, Sender}; - use crate::trace_processor::{self, TraceProcessor}; + use crate::{ + config::Config, + trace_processor::{self, TraceProcessor}, + }; use datadog_trace_protobuf::pb; fn create_test_span(start: i64, span_id: u64, parent_id: u64, is_top_level: bool) -> pb::Span { @@ -173,9 +196,12 @@ mod tests { }; if is_top_level { span.metrics.insert("_top_level".to_string(), 1.0); - span.meta.insert("functionname".to_string(), "".to_string()); span.meta .insert("_dd.origin".to_string(), "gcp_function".to_string()); + span.meta.insert( + "functionname".to_string(), + "dummy_function_name".to_string(), + ); span.r#type = "serverless".to_string(); } span @@ -211,6 +237,16 @@ mod tests { .as_nanos() as i64 } + fn create_test_config() -> Config { + Config { + api_key: "dummy_api_key".to_string(), + gcp_function_name: Some("dummy_function_name".to_string()), + max_request_content_length: 10 * 1024 * 1024, + trace_flush_interval: 3, + stats_flush_interval: 3, + } + } + #[tokio::test] async fn test_process_trace() { let (tx, mut rx): (Sender, Receiver) = @@ -227,11 +263,14 @@ mod tests { .header("datadog-meta-lang-version", "v19.7.0") .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") + .header("content-length", "100") .body(hyper::body::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; - let res = trace_processor.process_traces(request, tx).await; + let res = trace_processor + .process_traces(Arc::new(create_test_config()), request, tx) + .await; assert!(res.is_ok()); let tracer_payload = rx.recv().await; @@ -280,11 +319,14 @@ mod tests { .header("datadog-meta-lang-version", "v19.7.0") .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") + .header("content-length", "100") .body(hyper::body::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; - let res = trace_processor.process_traces(request, tx).await; + let res = trace_processor + .process_traces(Arc::new(create_test_config()), request, tx) + .await; assert!(res.is_ok()); let tracer_payload = rx.recv().await; From 8344dc178b7919eef5a5888e89e60e83b0c13b7c Mon Sep 17 00:00:00 2001 From: David Lee Date: Mon, 1 May 2023 16:23:06 -0700 Subject: [PATCH 05/64] Mini Agent: Verify GCP Env (#142) --- Cargo.toml | 2 + src/config.rs | 3 + src/env_verifier.rs | 292 +++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/mini_agent.rs | 33 ++++- src/trace_processor.rs | 23 +++- 6 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 src/env_verifier.rs diff --git a/Cargo.toml b/Cargo.toml index 87bf4de..c40e161 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,10 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] +anyhow = "1.0" hyper = { version = "0.14", default-features = false, features = ["server"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" log = "0.4" +serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } diff --git a/src/config.rs b/src/config.rs index 765f587..40c0cb7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,8 @@ pub struct Config { pub trace_flush_interval: u64, /// how often to flush stats, in seconds pub stats_flush_interval: u64, + /// timeout for environment verification, in milliseconds + pub verify_env_timeout: u64, } impl Config { @@ -33,6 +35,7 @@ impl Config { max_request_content_length: 10 * 1024 * 1024, // 10MB in Bytes trace_flush_interval: 3, stats_flush_interval: 3, + verify_env_timeout: 100, }) } } diff --git a/src/env_verifier.rs b/src/env_verifier.rs new file mode 100644 index 0000000..57d05c4 --- /dev/null +++ b/src/env_verifier.rs @@ -0,0 +1,292 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +use async_trait::async_trait; +use hyper::{Body, Client, Method, Request, Response}; +use log::{debug, error}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +#[cfg(not(test))] +use std::process; + +use datadog_trace_utils::trace_utils; + +const GCP_METADATA_URL: &str = "http://metadata.google.internal/computeMetadata/v1/?recursive=true"; + +#[derive(Default, Debug, Deserialize, Serialize)] +pub struct GCPMetadata { + pub instance: GCPInstance, + pub project: GCPProject, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct GCPInstance { + pub region: String, +} +impl Default for GCPInstance { + fn default() -> Self { + Self { + region: "unknown".to_string(), + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GCPProject { + pub project_id: String, +} +impl Default for GCPProject { + fn default() -> Self { + Self { + project_id: "unknown".to_string(), + } + } +} + +#[async_trait] +pub trait EnvVerifier { + /// Verifies the mini agent is running in the intended environment. if not, exit the process. + /// Returns MiniAgentMetadata, a struct of metadata collected from the environment. + async fn verify_environment(&self, verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata; +} + +pub struct ServerlessEnvVerifier {} + +#[async_trait] +impl EnvVerifier for ServerlessEnvVerifier { + async fn verify_environment(&self, verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata { + let gcp_metadata_request = + ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})); + let gcp_metadata = match tokio::time::timeout( + Duration::from_millis(verify_env_timeout), + gcp_metadata_request, + ) + .await + { + Ok(result) => match result { + Ok(res) => { + debug!("Successfully fetched Google Metadata."); + res + } + Err(e) => { + error!("The Mini Agent cannot be run in a non Google Cloud Function environment. Verification has failed, error: {e}. Shutting down now."); + #[cfg(not(test))] + process::exit(1); + #[cfg(test)] + GCPMetadata::default() + } + }, + Err(_) => { + error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); + GCPMetadata::default() + } + }; + trace_utils::MiniAgentMetadata { + gcp_project_id: Some(gcp_metadata.project.project_id), + gcp_region: Some(get_region_from_gcp_region_string( + gcp_metadata.instance.region, + )), + } + } +} + +/// The region found in GCP Metadata comes in the format: "projects/123123/regions/us-east1" +/// This function extracts just the region (us-east1) from this GCP region string. +/// If the string does not have 4 parts (separated by "/") or extraction fails, return "unknown" +fn get_region_from_gcp_region_string(str: String) -> String { + let split_str = str.split('/').collect::>(); + if split_str.len() != 4 { + return "unknown".to_string(); + } + match split_str.last() { + Some(res) => res.to_string(), + None => "unknown".to_string(), + } +} + +/// GoogleMetadataClient trait is used so we can mock a google metadata server response in unit tests +#[async_trait] +trait GoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result>; +} +struct GoogleMetadataClientWrapper {} + +#[async_trait] +impl GoogleMetadataClient for GoogleMetadataClientWrapper { + // async fn get_metadata(&self) -> anyhow::Result> { + async fn get_metadata(&self) -> anyhow::Result> { + let req = match Request::builder() + .method(Method::POST) + .uri(GCP_METADATA_URL) + .header("Metadata-Flavor", "Google") + .body(Body::empty()) + { + Ok(res) => res, + Err(err) => { + anyhow::bail!(err.to_string()) + } + }; + let client = Client::new(); + match client.request(req).await { + Ok(res) => Ok(res), + Err(err) => anyhow::bail!(err.to_string()), + } + } +} + +/// Checks if we are running in a Google Cloud Function environment. +/// If true, returns Metadata from the Google Cloud environment. +/// Otherwise, returns an error with the verification failure reason. +async fn ensure_gcp_function_environment( + metadata_client: Box, +) -> anyhow::Result { + let response = match metadata_client.get_metadata().await { + Ok(res) => res, + Err(e) => { + anyhow::bail!("Can't communicate with Google Metadata Server. {e}") + } + }; + let (parts, body) = response.into_parts(); + let headers = parts.headers; + match headers.get("Server") { + Some(val) => { + if val != "Metadata Server for Serverless" { + anyhow::bail!("In Google Cloud, but not in a function environment.") + } + } + None => { + anyhow::bail!("In Google Cloud, but server identifier not found.") + } + } + + let gcp_metadata = match get_gcp_metadata_from_body(body).await { + Ok(res) => res, + Err(err) => { + error!("Failed to get GCP Function Metadata. Will not enrich spans. {err}"); + return Ok(GCPMetadata::default()); + } + }; + + Ok(gcp_metadata) +} + +async fn get_gcp_metadata_from_body(body: hyper::Body) -> anyhow::Result { + let bytes = hyper::body::to_bytes(body).await?; + let body_str = String::from_utf8(bytes.to_vec())?; + let gcp_metadata: GCPMetadata = serde_json::from_str(&body_str)?; + Ok(gcp_metadata) +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use datadog_trace_utils::trace_utils::MiniAgentMetadata; + use hyper::{Body, Response, StatusCode}; + + use crate::env_verifier::{ + ensure_gcp_function_environment, get_region_from_gcp_region_string, GoogleMetadataClient, + }; + + use super::{EnvVerifier, GoogleMetadataClientWrapper, ServerlessEnvVerifier}; + + #[tokio::test] + async fn test_verify_env_false_if_metadata_server_unreachable() { + // unit tests will always run in an environment where http://metadata.google.internal is unreachable + let res = ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})).await; + assert!(res.is_err()); + assert!(res + .unwrap_err() + .to_string() + .contains("Can't communicate with Google Metadata Server. error trying to connect:")); + } + + #[tokio::test] + async fn test_verify_env_false_if_no_server_in_response_headers() { + struct MockGoogleMetadataClient {} + #[async_trait] + impl GoogleMetadataClient for MockGoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result> { + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::empty()) + .unwrap()) + } + } + let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + assert!(res.is_err()); + assert_eq!( + res.unwrap_err().to_string(), + "In Google Cloud, but server identifier not found." + ); + } + + #[tokio::test] + async fn test_verify_env_false_if_server_header_not_serverless() { + struct MockGoogleMetadataClient {} + #[async_trait] + impl GoogleMetadataClient for MockGoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result> { + Ok(Response::builder() + .status(StatusCode::OK) + .header("Server", "Metadata Server NOT for Serverless") + .body(Body::empty()) + .unwrap()) + } + } + let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + assert!(res.is_err()); + assert_eq!( + res.unwrap_err().to_string(), + "In Google Cloud, but not in a function environment." + ); + } + + #[tokio::test] + async fn test_verify_env_true_if_cloud_function_env() { + struct MockGoogleMetadataClient {} + #[async_trait] + impl GoogleMetadataClient for MockGoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result> { + Ok(Response::builder() + .status(StatusCode::OK) + .header("Server", "Metadata Server for Serverless") + .body(Body::empty()) + .unwrap()) + } + } + let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + assert!(res.is_ok()); + } + + #[tokio::test] + async fn test_verify_environment_timeout_exceeded_gives_unknown_values() { + let env_verifier = ServerlessEnvVerifier {}; + let res = env_verifier.verify_environment(0).await; // set the verify_env_timeout to timeout immediately + assert_eq!( + res, + MiniAgentMetadata { + gcp_project_id: Some("unknown".to_string()), + gcp_region: Some("unknown".to_string()) + } + ) + } + + #[test] + fn test_gcp_region_string_extraction_valid_string() { + let res = get_region_from_gcp_region_string("projects/123123/regions/us-east1".to_string()); + assert_eq!(res, "us-east1"); + } + + #[test] + fn test_gcp_region_string_extraction_wrong_number_of_parts() { + let res = get_region_from_gcp_region_string("invalid/parts/count".to_string()); + assert_eq!(res, "unknown"); + } + + #[test] + fn test_gcp_region_string_extraction_empty_string() { + let res = get_region_from_gcp_region_string("".to_string()); + assert_eq!(res, "unknown"); + } +} diff --git a/src/lib.rs b/src/lib.rs index ecec93a..3c9534f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. pub mod config; +pub mod env_verifier; pub mod http_utils; pub mod mini_agent; pub mod stats_flusher; diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 47f5f57..22c8c88 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -3,16 +3,18 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; -use log::{error, info}; +use log::{debug, error, info}; use serde_json::json; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; +use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; use crate::http_utils::log_and_create_http_response; -use crate::{config, stats_flusher, stats_processor, trace_flusher, trace_processor}; +use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; use datadog_trace_protobuf::pb; +use datadog_trace_utils::trace_utils; const MINI_AGENT_PORT: usize = 8126; const TRACE_ENDPOINT_PATH: &str = "/v0.4/traces"; @@ -27,11 +29,26 @@ pub struct MiniAgent { pub trace_flusher: Arc, pub stats_processor: Arc, pub stats_flusher: Arc, + pub env_verifier: Arc, } impl MiniAgent { #[tokio::main] pub async fn start_mini_agent(&self) -> Result<(), Box> { + let now = Instant::now(); + + // verify we are in a google cloud funtion environment. if not, shut down the mini agent. + let mini_agent_metadata = Arc::new( + self.env_verifier + .verify_environment(self.config.verify_env_timeout) + .await, + ); + + debug!( + "Time taken to fetch Mini Agent metadata: {} ms", + now.elapsed().as_millis() + ); + // setup a channel to send processed traces to our flusher. tx is passed through each endpoint_handler // to the trace processor, which uses it to send de-serialized processed trace payloads to our trace flusher. let (trace_tx, trace_rx): (Sender, Receiver) = @@ -76,6 +93,7 @@ impl MiniAgent { let stats_tx = stats_tx.clone(); let endpoint_config = endpoint_config.clone(); + let mini_agent_metadata = Arc::clone(&mini_agent_metadata); let service = service_fn(move |req| { MiniAgent::trace_endpoint_handler( @@ -85,6 +103,7 @@ impl MiniAgent { trace_tx.clone(), stats_processor.clone(), stats_tx.clone(), + Arc::clone(&mini_agent_metadata), ) }); @@ -97,6 +116,10 @@ impl MiniAgent { let server = server_builder.serve(make_svc); info!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); + debug!( + "Time taken start the Mini Agent: {} ms", + now.elapsed().as_millis() + ); // start hyper http server if let Err(e) = server.await { @@ -114,10 +137,14 @@ impl MiniAgent { trace_tx: Sender, stats_processor: Arc, stats_tx: Sender, + mini_agent_metadata: Arc, ) -> http::Result> { match (req.method(), req.uri().path()) { (&Method::PUT, TRACE_ENDPOINT_PATH) => { - match trace_processor.process_traces(config, req, trace_tx).await { + match trace_processor + .process_traces(config, req, trace_tx, mini_agent_metadata) + .await + { Ok(res) => Ok(res), Err(err) => log_and_create_http_response( &format!("Error processing traces: {err}"), diff --git a/src/trace_processor.rs b/src/trace_processor.rs index c775019..76f1257 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -39,6 +39,7 @@ pub trait TraceProcessor { config: Arc, req: Request, tx: Sender, + mini_agent_metadata: Arc, ) -> http::Result>; } @@ -52,6 +53,7 @@ impl TraceProcessor for ServerlessTraceProcessor { config: Arc, req: Request, tx: Sender, + mini_agent_metadata: Arc, ) -> http::Result> { let (parts, body) = req.into_parts(); @@ -105,6 +107,7 @@ impl TraceProcessor for ServerlessTraceProcessor { if tracer_header_tags.client_computed_top_level { trace_utils::update_tracer_top_level(span); } + trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); } if !tracer_header_tags.client_computed_top_level { @@ -170,6 +173,7 @@ mod tests { trace_processor::{self, TraceProcessor}, }; use datadog_trace_protobuf::pb; + use datadog_trace_utils::trace_utils; fn create_test_span(start: i64, span_id: u64, parent_id: u64, is_top_level: bool) -> pb::Span { let mut span = pb::Span { @@ -197,7 +201,9 @@ mod tests { if is_top_level { span.metrics.insert("_top_level".to_string(), 1.0); span.meta - .insert("_dd.origin".to_string(), "gcp_function".to_string()); + .insert("_dd.origin".to_string(), "cloudfunction".to_string()); + span.meta + .insert("origin".to_string(), "cloudfunction".to_string()); span.meta.insert( "functionname".to_string(), "dummy_function_name".to_string(), @@ -244,6 +250,7 @@ mod tests { max_request_content_length: 10 * 1024 * 1024, trace_flush_interval: 3, stats_flush_interval: 3, + verify_env_timeout: 100, } } @@ -269,7 +276,12 @@ mod tests { let trace_processor = trace_processor::ServerlessTraceProcessor {}; let res = trace_processor - .process_traces(Arc::new(create_test_config()), request, tx) + .process_traces( + Arc::new(create_test_config()), + request, + tx, + Arc::new(trace_utils::MiniAgentMetadata::default()), + ) .await; assert!(res.is_ok()); @@ -325,7 +337,12 @@ mod tests { let trace_processor = trace_processor::ServerlessTraceProcessor {}; let res = trace_processor - .process_traces(Arc::new(create_test_config()), request, tx) + .process_traces( + Arc::new(create_test_config()), + request, + tx, + Arc::new(trace_utils::MiniAgentMetadata::default()), + ) .await; assert!(res.is_ok()); From 8f43c43e70381afc79a639367c839557609ef3ce Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 2 May 2023 09:21:44 -0700 Subject: [PATCH 06/64] Fix env_verifier + protobuf gitlab build issues (#148) --- src/env_verifier.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 57d05c4..421b001 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -188,17 +188,23 @@ mod tests { ensure_gcp_function_environment, get_region_from_gcp_region_string, GoogleMetadataClient, }; - use super::{EnvVerifier, GoogleMetadataClientWrapper, ServerlessEnvVerifier}; + use super::{EnvVerifier, ServerlessEnvVerifier}; #[tokio::test] async fn test_verify_env_false_if_metadata_server_unreachable() { - // unit tests will always run in an environment where http://metadata.google.internal is unreachable - let res = ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})).await; + struct MockGoogleMetadataClient {} + #[async_trait] + impl GoogleMetadataClient for MockGoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result> { + anyhow::bail!("Random Error") + } + } + let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; assert!(res.is_err()); assert!(res .unwrap_err() .to_string() - .contains("Can't communicate with Google Metadata Server. error trying to connect:")); + .contains("Can't communicate with Google Metadata Server")); } #[tokio::test] From cd6eee2bb3e7439337de0809bc972de2c842bd79 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 3 May 2023 15:17:03 -0700 Subject: [PATCH 07/64] Mini Agent: Support DD_SITE (#149) --- Cargo.toml | 4 +- src/config.rs | 104 ++++++++++++++++++++++++++++++++++++++++- src/stats_flusher.rs | 8 +++- src/trace_flusher.rs | 8 +++- src/trace_processor.rs | 3 ++ 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c40e161..34c2516 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,4 +16,6 @@ datadog-trace-utils = { path = "../trace-utils" } datadog-trace-normalization = { path = "../trace-normalization" } [dev-dependencies] -rmp-serde = "1.1.1" \ No newline at end of file +rmp-serde = "1.1.1" +serial_test = "2.0.0" +duplicate = "0.4.1" diff --git a/src/config.rs b/src/config.rs index 40c0cb7..db3198f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,6 +3,9 @@ use std::env; +const TRACE_INTAKE_ROUTE: &str = "/api/v0.2/traces"; +const TRACE_STATS_INTAKE_ROUTE: &str = "/api/v0.2/stats"; + #[derive(Debug, Clone)] pub struct Config { pub api_key: String, @@ -14,11 +17,15 @@ pub struct Config { pub stats_flush_interval: u64, /// timeout for environment verification, in milliseconds pub verify_env_timeout: u64, + pub trace_intake_url: String, + pub trace_stats_intake_url: String, + pub dd_site: String, } impl Config { pub fn new() -> Result> { - let api_key = env::var("DD_API_KEY")?; + let api_key = env::var("DD_API_KEY") + .map_err(|_| anyhow::anyhow!("DD_API_KEY environment variable is not set"))?; let mut function_name = None; // Google cloud functions automatically sets either K_SERVICE or FUNCTION_NAME @@ -29,6 +36,12 @@ impl Config { } else if let Ok(res) = env::var("FUNCTION_NAME") { function_name = Some(res); } + + let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); + + let trace_intake_url = construct_trace_intake_url(&dd_site, TRACE_INTAKE_ROUTE); + let trace_stats_intake_url = construct_trace_intake_url(&dd_site, TRACE_STATS_INTAKE_ROUTE); + Ok(Config { api_key, gcp_function_name: function_name, @@ -36,6 +49,95 @@ impl Config { trace_flush_interval: 3, stats_flush_interval: 3, verify_env_timeout: 100, + dd_site, + trace_intake_url, + trace_stats_intake_url, }) } } + +fn construct_trace_intake_url(prefix: &str, route: &str) -> String { + format!("https://trace.agent.{prefix}{route}") +} + +#[cfg(test)] +mod tests { + use duplicate::duplicate_item; + use serial_test::serial; + use std::env; + + use crate::config; + + #[test] + #[serial] + fn test_error_if_no_api_key_env_var() { + let config = config::Config::new(); + assert!(config.is_err()); + assert_eq!( + config.unwrap_err().to_string(), + "DD_API_KEY environment variable is not set" + ); + } + + #[test] + #[serial] + fn test_default_trace_and_trace_stats_urls() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.trace_intake_url, + "https://trace.agent.datadoghq.com/api/v0.2/traces" + ); + assert_eq!( + config.trace_stats_intake_url, + "https://trace.agent.datadoghq.com/api/v0.2/stats" + ); + env::remove_var("DD_API_KEY"); + } + + #[duplicate_item( + test_name dd_site expected_url; + [test_us1_trace_intake_url] ["datadoghq.com"] ["https://trace.agent.datadoghq.com/api/v0.2/traces"]; + [test_us3_trace_intake_url] ["us3.datadoghq.com"] ["https://trace.agent.us3.datadoghq.com/api/v0.2/traces"]; + [test_us5_trace_intake_url] ["us5.datadoghq.com"] ["https://trace.agent.us5.datadoghq.com/api/v0.2/traces"]; + [test_eu_trace_intake_url] ["datadoghq.eu"] ["https://trace.agent.datadoghq.eu/api/v0.2/traces"]; + [test_ap1_trace_intake_url] ["ap1.datadoghq.com"] ["https://trace.agent.ap1.datadoghq.com/api/v0.2/traces"]; + [test_gov_trace_intake_url] ["ddog-gov.com"] ["https://trace.agent.ddog-gov.com/api/v0.2/traces"]; + )] + #[test] + #[serial] + fn test_name() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("DD_SITE", dd_site); + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.trace_intake_url, expected_url); + env::remove_var("DD_API_KEY"); + env::remove_var("DD_SITE"); + } + + #[duplicate_item( + test_name dd_site expected_url; + [test_us1_trace_stats_intake_url] ["datadoghq.com"] ["https://trace.agent.datadoghq.com/api/v0.2/stats"]; + [test_us3_trace_stats_intake_url] ["us3.datadoghq.com"] ["https://trace.agent.us3.datadoghq.com/api/v0.2/stats"]; + [test_us5_trace_stats_intake_url] ["us5.datadoghq.com"] ["https://trace.agent.us5.datadoghq.com/api/v0.2/stats"]; + [test_eu_trace_stats_intake_url] ["datadoghq.eu"] ["https://trace.agent.datadoghq.eu/api/v0.2/stats"]; + [test_ap1_trace_stats_intake_url] ["ap1.datadoghq.com"] ["https://trace.agent.ap1.datadoghq.com/api/v0.2/stats"]; + [test_gov_trace_stats_intake_url] ["ddog-gov.com"] ["https://trace.agent.ddog-gov.com/api/v0.2/stats"]; + )] + #[test] + #[serial] + fn test_name() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("DD_SITE", dd_site); + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.trace_stats_intake_url, expected_url); + env::remove_var("DD_API_KEY"); + env::remove_var("DD_SITE"); + } +} diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index 20f60bc..badda1e 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -75,7 +75,13 @@ impl StatsFlusher for ServerlessStatsFlusher { } }; - match stats_utils::send_stats_payload(serialized_stats_payload, &config.api_key).await { + match stats_utils::send_stats_payload( + serialized_stats_payload, + &config.trace_stats_intake_url, + &config.api_key, + ) + .await + { Ok(_) => info!("Successfully flushed stats"), Err(e) => { error!("Error sending stats: {e:?}") diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index ec4c06e..8f59a72 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -67,7 +67,13 @@ impl TraceFlusher for ServerlessTraceFlusher { } }; - match trace_utils::send(serialized_agent_payload, &config.api_key).await { + match trace_utils::send( + serialized_agent_payload, + &config.trace_intake_url, + &config.api_key, + ) + .await + { Ok(_) => info!("Successfully flushed traces"), Err(e) => { error!("Error sending trace: {e:?}") diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 76f1257..4312a24 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -251,6 +251,9 @@ mod tests { trace_flush_interval: 3, stats_flush_interval: 3, verify_env_timeout: 100, + trace_intake_url: "trace.agent.notdog.com/traces".to_string(), + trace_stats_intake_url: "trace.agent.notdog.com/stats".to_string(), + dd_site: "datadoghq.com".to_string(), } } From 194aa970225d5ec1c5fbc0c06254b7fc7c61f9eb Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 1 Jun 2023 10:10:08 -0700 Subject: [PATCH 08/64] Mini Agent: Support env var set custom trace & trace stats intake urls (#172) --- src/config.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/config.rs b/src/config.rs index db3198f..4690e70 100644 --- a/src/config.rs +++ b/src/config.rs @@ -39,8 +39,17 @@ impl Config { let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); - let trace_intake_url = construct_trace_intake_url(&dd_site, TRACE_INTAKE_ROUTE); - let trace_stats_intake_url = construct_trace_intake_url(&dd_site, TRACE_STATS_INTAKE_ROUTE); + // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & trace stats to) + let mut trace_intake_url = format!("https://trace.agent.{dd_site}{TRACE_INTAKE_ROUTE}"); + let mut trace_stats_intake_url = + format!("https://trace.agent.{dd_site}{TRACE_STATS_INTAKE_ROUTE}"); + + // DD_APM_DD_URL env var will primarily be used for integration tests + // overrides the entire trace/trace stats intake url prefix + if let Ok(endpoint_prefix) = env::var("DD_APM_DD_URL") { + trace_intake_url = format!("{endpoint_prefix}{TRACE_INTAKE_ROUTE}"); + trace_stats_intake_url = format!("{endpoint_prefix}{TRACE_STATS_INTAKE_ROUTE}"); + }; Ok(Config { api_key, @@ -56,10 +65,6 @@ impl Config { } } -fn construct_trace_intake_url(prefix: &str, route: &str) -> String { - format!("https://trace.agent.{prefix}{route}") -} - #[cfg(test)] mod tests { use duplicate::duplicate_item; @@ -140,4 +145,24 @@ mod tests { env::remove_var("DD_API_KEY"); env::remove_var("DD_SITE"); } + + #[test] + #[serial] + fn test_set_custom_trace_and_trace_stats_intake_url() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("DD_APM_DD_URL", "http://127.0.0.1:3333"); + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.trace_intake_url, + "http://127.0.0.1:3333/api/v0.2/traces" + ); + assert_eq!( + config.trace_stats_intake_url, + "http://127.0.0.1:3333/api/v0.2/stats" + ); + env::remove_var("DD_API_KEY"); + env::remove_var("DD_APM_DD_URL"); + } } From 5bc563c5dc371349ff56aa788658d13cd4784708 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 13 Jun 2023 10:44:19 -0700 Subject: [PATCH 09/64] Mini Agent: Verify azure function env w/ filesystem (#174) --- Cargo.toml | 1 + src/config.rs | 48 +++++- src/env_verifier.rs | 376 +++++++++++++++++++++++++++++++++-------- src/mini_agent.rs | 6 +- src/trace_processor.rs | 7 +- 5 files changed, 363 insertions(+), 75 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 34c2516..ee58ef1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ datadog-trace-normalization = { path = "../trace-normalization" } rmp-serde = "1.1.1" serial_test = "2.0.0" duplicate = "0.4.1" +tempfile = "3.3.0" diff --git a/src/config.rs b/src/config.rs index 4690e70..bd108fa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,13 +3,17 @@ use std::env; +use datadog_trace_utils::trace_utils; + const TRACE_INTAKE_ROUTE: &str = "/api/v0.2/traces"; const TRACE_STATS_INTAKE_ROUTE: &str = "/api/v0.2/stats"; #[derive(Debug, Clone)] pub struct Config { pub api_key: String, - pub gcp_function_name: Option, + pub function_name: Option, + pub env_type: trace_utils::EnvironmentType, + pub os: String, pub max_request_content_length: usize, /// how often to flush traces, in seconds pub trace_flush_interval: u64, @@ -28,15 +32,25 @@ impl Config { .map_err(|_| anyhow::anyhow!("DD_API_KEY environment variable is not set"))?; let mut function_name = None; - // Google cloud functions automatically sets either K_SERVICE or FUNCTION_NAME - // env vars to denote the cloud function name. - // K_SERVICE is set on newer runtimes, while FUNCTION_NAME is set on older deprecated runtimes. + let mut maybe_env_type = None; if let Ok(res) = env::var("K_SERVICE") { + // Set by Google Cloud Functions for newer runtimes function_name = Some(res); + maybe_env_type = Some(trace_utils::EnvironmentType::CloudFunction); } else if let Ok(res) = env::var("FUNCTION_NAME") { + // Set by Google Cloud Functions for older runtimes + function_name = Some(res); + maybe_env_type = Some(trace_utils::EnvironmentType::CloudFunction); + } else if let Ok(res) = env::var("WEBSITE_SITE_NAME") { + // Set by Azure Functions function_name = Some(res); + maybe_env_type = Some(trace_utils::EnvironmentType::AzureFunction); } + let env_type = maybe_env_type.ok_or_else(|| { + anyhow::anyhow!("Unable to identify environment. Shutting down Mini Agent.") + })?; + let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & trace stats to) @@ -53,7 +67,9 @@ impl Config { Ok(Config { api_key, - gcp_function_name: function_name, + function_name, + env_type, + os: env::consts::OS.to_string(), max_request_content_length: 10 * 1024 * 1024, // 10MB in Bytes trace_flush_interval: 3, stats_flush_interval: 3, @@ -73,6 +89,20 @@ mod tests { use crate::config; + #[test] + #[serial] + fn test_error_if_unable_to_identify_env() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + + let config = config::Config::new(); + assert!(config.is_err()); + assert_eq!( + config.unwrap_err().to_string(), + "Unable to identify environment. Shutting down Mini Agent." + ); + env::remove_var("DD_API_KEY"); + } + #[test] #[serial] fn test_error_if_no_api_key_env_var() { @@ -88,6 +118,7 @@ mod tests { #[serial] fn test_default_trace_and_trace_stats_urls() { env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("K_SERVICE", "function_name"); let config_res = config::Config::new(); assert!(config_res.is_ok()); let config = config_res.unwrap(); @@ -100,6 +131,7 @@ mod tests { "https://trace.agent.datadoghq.com/api/v0.2/stats" ); env::remove_var("DD_API_KEY"); + env::remove_var("K_SERVICE"); } #[duplicate_item( @@ -115,6 +147,7 @@ mod tests { #[serial] fn test_name() { env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("K_SERVICE", "function_name"); env::set_var("DD_SITE", dd_site); let config_res = config::Config::new(); assert!(config_res.is_ok()); @@ -122,6 +155,7 @@ mod tests { assert_eq!(config.trace_intake_url, expected_url); env::remove_var("DD_API_KEY"); env::remove_var("DD_SITE"); + env::remove_var("K_SERVICE"); } #[duplicate_item( @@ -137,6 +171,7 @@ mod tests { #[serial] fn test_name() { env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("K_SERVICE", "function_name"); env::set_var("DD_SITE", dd_site); let config_res = config::Config::new(); assert!(config_res.is_ok()); @@ -144,12 +179,14 @@ mod tests { assert_eq!(config.trace_stats_intake_url, expected_url); env::remove_var("DD_API_KEY"); env::remove_var("DD_SITE"); + env::remove_var("K_SERVICE"); } #[test] #[serial] fn test_set_custom_trace_and_trace_stats_intake_url() { env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("K_SERVICE", "function_name"); env::set_var("DD_APM_DD_URL", "http://127.0.0.1:3333"); let config_res = config::Config::new(); assert!(config_res.is_ok()); @@ -164,5 +201,6 @@ mod tests { ); env::remove_var("DD_API_KEY"); env::remove_var("DD_APM_DD_URL"); + env::remove_var("K_SERVICE"); } } diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 421b001..d8f6aa4 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -4,22 +4,26 @@ use async_trait::async_trait; use hyper::{Body, Client, Method, Request, Response}; use log::{debug, error}; use serde::{Deserialize, Serialize}; -use std::time::Duration; - -#[cfg(not(test))] +use std::fs; +use std::path::Path; use std::process; +use std::time::{Duration, Instant}; use datadog_trace_utils::trace_utils; const GCP_METADATA_URL: &str = "http://metadata.google.internal/computeMetadata/v1/?recursive=true"; +const AZURE_LINUX_FUNCTION_ROOT_PATH_STR: &str = "/home/site/wwwroot"; +const AZURE_WINDOWS_FUNCTION_ROOT_PATH_STR: &str = "C:\\home\\site\\wwwroot"; +const AZURE_HOST_JSON_NAME: &str = "host.json"; +const AZURE_FUNCTION_JSON_NAME: &str = "function.json"; -#[derive(Default, Debug, Deserialize, Serialize)] +#[derive(Default, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct GCPMetadata { pub instance: GCPInstance, pub project: GCPProject, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct GCPInstance { pub region: String, } @@ -31,7 +35,7 @@ impl Default for GCPInstance { } } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GCPProject { pub project_id: String, @@ -48,46 +52,65 @@ impl Default for GCPProject { pub trait EnvVerifier { /// Verifies the mini agent is running in the intended environment. if not, exit the process. /// Returns MiniAgentMetadata, a struct of metadata collected from the environment. - async fn verify_environment(&self, verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata; + async fn verify_environment( + &self, + verify_env_timeout: u64, + env_type: &trace_utils::EnvironmentType, + os: &str, + ) -> trace_utils::MiniAgentMetadata; } pub struct ServerlessEnvVerifier {} #[async_trait] impl EnvVerifier for ServerlessEnvVerifier { - async fn verify_environment(&self, verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata { - let gcp_metadata_request = - ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})); - let gcp_metadata = match tokio::time::timeout( - Duration::from_millis(verify_env_timeout), - gcp_metadata_request, - ) - .await - { - Ok(result) => match result { - Ok(res) => { - debug!("Successfully fetched Google Metadata."); - res - } - Err(e) => { - error!("The Mini Agent cannot be run in a non Google Cloud Function environment. Verification has failed, error: {e}. Shutting down now."); - #[cfg(not(test))] - process::exit(1); - #[cfg(test)] - GCPMetadata::default() - } - }, - Err(_) => { - error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); - GCPMetadata::default() + async fn verify_environment( + &self, + verify_env_timeout: u64, + env_type: &trace_utils::EnvironmentType, + os: &str, + ) -> trace_utils::MiniAgentMetadata { + match env_type { + trace_utils::EnvironmentType::AzureFunction => { + verify_azure_environment_or_exit(os).await; + trace_utils::MiniAgentMetadata::default() + } + trace_utils::EnvironmentType::CloudFunction => { + return verify_gcp_environment_or_exit(verify_env_timeout).await; + } + } + } +} + +async fn verify_gcp_environment_or_exit(verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata { + let gcp_metadata_request = + ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})); + let gcp_metadata = match tokio::time::timeout( + Duration::from_millis(verify_env_timeout), + gcp_metadata_request, + ) + .await + { + Ok(result) => match result { + Ok(metadata) => { + debug!("Successfully fetched Google Metadata."); + metadata } - }; - trace_utils::MiniAgentMetadata { - gcp_project_id: Some(gcp_metadata.project.project_id), - gcp_region: Some(get_region_from_gcp_region_string( - gcp_metadata.instance.region, - )), + Err(err) => { + error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {err}"); + process::exit(1); + } + }, + Err(_) => { + error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); + GCPMetadata::default() } + }; + trace_utils::MiniAgentMetadata { + gcp_project_id: Some(gcp_metadata.project.project_id), + gcp_region: Some(get_region_from_gcp_region_string( + gcp_metadata.instance.region, + )), } } @@ -114,19 +137,14 @@ struct GoogleMetadataClientWrapper {} #[async_trait] impl GoogleMetadataClient for GoogleMetadataClientWrapper { - // async fn get_metadata(&self) -> anyhow::Result> { async fn get_metadata(&self) -> anyhow::Result> { - let req = match Request::builder() + let req = Request::builder() .method(Method::POST) .uri(GCP_METADATA_URL) .header("Metadata-Flavor", "Google") .body(Body::empty()) - { - Ok(res) => res, - Err(err) => { - anyhow::bail!(err.to_string()) - } - }; + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + let client = Client::new(); match client.request(req).await { Ok(res) => Ok(res), @@ -141,12 +159,10 @@ impl GoogleMetadataClient for GoogleMetadataClientWrapper { async fn ensure_gcp_function_environment( metadata_client: Box, ) -> anyhow::Result { - let response = match metadata_client.get_metadata().await { - Ok(res) => res, - Err(e) => { - anyhow::bail!("Can't communicate with Google Metadata Server. {e}") - } - }; + let response = metadata_client.get_metadata().await.map_err(|err| { + anyhow::anyhow!("Can't communicate with Google Metadata Server. Error: {err}") + })?; + let (parts, body) = response.into_parts(); let headers = parts.headers; match headers.get("Server") { @@ -178,20 +194,112 @@ async fn get_gcp_metadata_from_body(body: hyper::Body) -> anyhow::Result { + debug!("Successfully verified Azure Function Environment."); + } + Err(e) => { + error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {e}"); + process::exit(1); + } + } + debug!( + "Time taken to verify Azure Functions env: {} ms", + now.elapsed().as_millis() + ); +} + +/// AzureVerificationClient trait is used so we can mock the azure function local url response in unit tests +trait AzureVerificationClient { + fn get_function_root_files(&self, path: &Path) -> anyhow::Result>; +} +struct AzureVerificationClientWrapper {} + +impl AzureVerificationClient for AzureVerificationClientWrapper { + fn get_function_root_files(&self, path: &Path) -> anyhow::Result> { + let mut file_names: Vec = Vec::new(); + + let entries = fs::read_dir(path)?; + for entry in entries { + let entry = entry.map_err(|e| anyhow::anyhow!(e))?; + let entry_name = entry.file_name(); + if entry_name == "node_modules" { + continue; + } + + file_names.push(entry_name.to_string_lossy().to_string()); + + if entry.file_type()?.is_dir() { + let sub_entries = fs::read_dir(entry.path())?; + for sub_entry in sub_entries { + let sub_entry = sub_entry.map_err(|e| anyhow::anyhow!(e))?; + let sub_entry_name = sub_entry.file_name(); + file_names.push(sub_entry_name.to_string_lossy().to_string()); + } + } + } + Ok(file_names) + } +} + +/// Checks if we are running in an Azure Function environment. +/// If true, returns MiniAgentMetadata default. +/// Otherwise, returns an error with the verification failure reason. +async fn ensure_azure_function_environment( + verification_client: Box, + os: &str, +) -> anyhow::Result<()> { + let azure_linux_function_root_path = Path::new(AZURE_LINUX_FUNCTION_ROOT_PATH_STR); + let azure_windows_function_root_path = Path::new(AZURE_WINDOWS_FUNCTION_ROOT_PATH_STR); + let function_files = match os { + "linux" => verification_client.get_function_root_files(azure_linux_function_root_path), + "windows" => verification_client.get_function_root_files(azure_windows_function_root_path), + _ => { + anyhow::bail!("The Serverless Mini Agent does not support this platform.") + } + }; + + let function_files = function_files.map_err(|e| anyhow::anyhow!(e))?; + + let mut host_json_exists = false; + let mut function_json_exists = false; + for file in function_files { + if file == AZURE_HOST_JSON_NAME { + host_json_exists = true; + } + if file == AZURE_FUNCTION_JSON_NAME { + function_json_exists = true; + } + } + + if !host_json_exists && !function_json_exists { + anyhow::bail!("Failed to validate an Azure Function directory system."); + } + Ok(()) +} + #[cfg(test)] mod tests { use async_trait::async_trait; - use datadog_trace_utils::trace_utils::MiniAgentMetadata; + use datadog_trace_utils::trace_utils; use hyper::{Body, Response, StatusCode}; + use serde_json::json; + use serial_test::serial; + use std::{fs, path::Path}; use crate::env_verifier::{ - ensure_gcp_function_environment, get_region_from_gcp_region_string, GoogleMetadataClient, + ensure_azure_function_environment, ensure_gcp_function_environment, + get_region_from_gcp_region_string, AzureVerificationClient, AzureVerificationClientWrapper, + GCPInstance, GCPMetadata, GCPProject, GoogleMetadataClient, AZURE_FUNCTION_JSON_NAME, + AZURE_HOST_JSON_NAME, }; use super::{EnvVerifier, ServerlessEnvVerifier}; #[tokio::test] - async fn test_verify_env_false_if_metadata_server_unreachable() { + async fn test_ensure_gcp_env_false_if_metadata_server_unreachable() { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { @@ -201,14 +309,14 @@ mod tests { } let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; assert!(res.is_err()); - assert!(res - .unwrap_err() - .to_string() - .contains("Can't communicate with Google Metadata Server")); + assert_eq!( + res.unwrap_err().to_string(), + "Can't communicate with Google Metadata Server. Error: Random Error" + ); } #[tokio::test] - async fn test_verify_env_false_if_no_server_in_response_headers() { + async fn test_ensure_gcp_env_false_if_no_server_in_response_headers() { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { @@ -228,7 +336,7 @@ mod tests { } #[tokio::test] - async fn test_verify_env_false_if_server_header_not_serverless() { + async fn test_ensure_gcp_env_if_server_header_not_serverless() { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { @@ -249,7 +357,7 @@ mod tests { } #[tokio::test] - async fn test_verify_env_true_if_cloud_function_env() { + async fn test_ensure_gcp_env_true_if_cloud_function_env() { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { @@ -257,23 +365,46 @@ mod tests { Ok(Response::builder() .status(StatusCode::OK) .header("Server", "Metadata Server for Serverless") - .body(Body::empty()) + .body(Body::from( + json!({ + "instance": { + "region": "projects/123123/regions/us-east1", + }, + "project": { + "projectId": "my-project" + } + }) + .to_string(), + )) .unwrap()) } } let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; assert!(res.is_ok()); + assert_eq!( + res.unwrap(), + GCPMetadata { + instance: GCPInstance { + region: "projects/123123/regions/us-east1".to_string() + }, + project: GCPProject { + project_id: "my-project".to_string() + } + } + ) } #[tokio::test] - async fn test_verify_environment_timeout_exceeded_gives_unknown_values() { + async fn test_gcp_verify_environment_timeout_exceeded_gives_unknown_values() { let env_verifier = ServerlessEnvVerifier {}; - let res = env_verifier.verify_environment(0).await; // set the verify_env_timeout to timeout immediately + let res = env_verifier + .verify_environment(0, &trace_utils::EnvironmentType::CloudFunction, "linux") + .await; // set the verify_env_timeout to timeout immediately assert_eq!( res, - MiniAgentMetadata { + trace_utils::MiniAgentMetadata { gcp_project_id: Some("unknown".to_string()), - gcp_region: Some("unknown".to_string()) + gcp_region: Some("unknown".to_string()), } ) } @@ -295,4 +426,115 @@ mod tests { let res = get_region_from_gcp_region_string("".to_string()); assert_eq!(res, "unknown"); } + + #[tokio::test] + async fn test_ensure_azure_env_windows_true() { + struct MockAzureVerificationClient {} + #[async_trait] + impl AzureVerificationClient for MockAzureVerificationClient { + fn get_function_root_files(&self, _path: &Path) -> anyhow::Result> { + Ok(vec!["host.json".to_string(), "function.json".to_string()]) + } + } + let res = + ensure_azure_function_environment(Box::new(MockAzureVerificationClient {}), "windows") + .await; + assert!(res.is_ok()); + } + + #[tokio::test] + async fn test_ensure_azure_env_windows_false() { + struct MockAzureVerificationClient {} + #[async_trait] + impl AzureVerificationClient for MockAzureVerificationClient { + fn get_function_root_files(&self, _path: &Path) -> anyhow::Result> { + Ok(vec![ + "random_file.json".to_string(), + "random_file_1.json".to_string(), + ]) + } + } + let res = + ensure_azure_function_environment(Box::new(MockAzureVerificationClient {}), "windows") + .await; + assert!(res.is_err()); + assert_eq!( + res.unwrap_err().to_string(), + "Failed to validate an Azure Function directory system." + ); + } + + #[tokio::test] + async fn test_ensure_azure_env_linux_true() { + struct MockAzureVerificationClient {} + #[async_trait] + impl AzureVerificationClient for MockAzureVerificationClient { + fn get_function_root_files(&self, _path: &Path) -> anyhow::Result> { + Ok(vec!["host.json".to_string(), "function.json".to_string()]) + } + } + let res = + ensure_azure_function_environment(Box::new(MockAzureVerificationClient {}), "linux") + .await; + assert!(res.is_ok()); + } + + #[tokio::test] + async fn test_ensure_azure_env_linux_false() { + struct MockAzureVerificationClient {} + #[async_trait] + impl AzureVerificationClient for MockAzureVerificationClient { + fn get_function_root_files(&self, _path: &Path) -> anyhow::Result> { + Ok(vec![ + "random_file.json".to_string(), + "random_file_1.json".to_string(), + ]) + } + } + let res = + ensure_azure_function_environment(Box::new(MockAzureVerificationClient {}), "linux") + .await; + assert!(res.is_err()); + assert_eq!( + res.unwrap_err().to_string(), + "Failed to validate an Azure Function directory system." + ); + } + + #[test] + #[serial] + fn test_get_function_root_files_returns_correct_files() { + let temp_dir = tempfile::tempdir().unwrap(); + let temp_dir_path = temp_dir.path(); + + fs::File::create(temp_dir_path.join(AZURE_HOST_JSON_NAME)).unwrap(); + fs::create_dir(temp_dir_path.join("HttpTrigger1")).unwrap(); + fs::File::create(temp_dir_path.join(format!("HttpTrigger1/{AZURE_FUNCTION_JSON_NAME}"))) + .unwrap(); + + let client = AzureVerificationClientWrapper {}; + + let files = client.get_function_root_files(temp_dir_path).unwrap(); + + assert!(files.contains(&AZURE_HOST_JSON_NAME.to_string())); + assert!(files.contains(&AZURE_FUNCTION_JSON_NAME.to_string())); + assert!(files.contains(&"HttpTrigger1".to_string())); + } + + #[test] + #[serial] + fn test_get_function_root_files_ignores_node_modules() { + let temp_dir = tempfile::tempdir().unwrap(); + let temp_dir_path = temp_dir.path(); + + fs::File::create(temp_dir_path.join(AZURE_HOST_JSON_NAME)).unwrap(); + fs::create_dir(temp_dir_path.join("node_modules")).unwrap(); + fs::File::create(temp_dir_path.join("node_modules/random.txt")).unwrap(); + + let client = AzureVerificationClientWrapper {}; + + let files = client.get_function_root_files(temp_dir_path).unwrap(); + + assert_eq!(files, vec![AZURE_HOST_JSON_NAME]); + } } diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 22c8c88..196d1a2 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -40,7 +40,11 @@ impl MiniAgent { // verify we are in a google cloud funtion environment. if not, shut down the mini agent. let mini_agent_metadata = Arc::new( self.env_verifier - .verify_environment(self.config.verify_env_timeout) + .verify_environment( + self.config.verify_env_timeout, + &self.config.env_type, + &self.config.os, + ) .await, ); diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 4312a24..9ef6268 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -116,7 +116,8 @@ impl TraceProcessor for ServerlessTraceProcessor { trace_utils::set_serverless_root_span_tags( &mut chunk.spans[root_span_index], - config.gcp_function_name.clone(), + config.function_name.clone(), + &config.env_type, ); trace_chunks.push(chunk); @@ -246,7 +247,7 @@ mod tests { fn create_test_config() -> Config { Config { api_key: "dummy_api_key".to_string(), - gcp_function_name: Some("dummy_function_name".to_string()), + function_name: Some("dummy_function_name".to_string()), max_request_content_length: 10 * 1024 * 1024, trace_flush_interval: 3, stats_flush_interval: 3, @@ -254,6 +255,8 @@ mod tests { trace_intake_url: "trace.agent.notdog.com/traces".to_string(), trace_stats_intake_url: "trace.agent.notdog.com/stats".to_string(), dd_site: "datadoghq.com".to_string(), + env_type: trace_utils::EnvironmentType::CloudFunction, + os: "linux".to_string(), } } From f73ef138624378f20cdca73aa17f0d0bdf4a3ad8 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 13 Jun 2023 12:30:45 -0700 Subject: [PATCH 10/64] support dotnet (#177) --- src/mini_agent.rs | 4 ++-- src/stats_processor.rs | 3 +-- src/trace_processor.rs | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 196d1a2..774297e 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -144,7 +144,7 @@ impl MiniAgent { mini_agent_metadata: Arc, ) -> http::Result> { match (req.method(), req.uri().path()) { - (&Method::PUT, TRACE_ENDPOINT_PATH) => { + (&Method::PUT | &Method::POST, TRACE_ENDPOINT_PATH) => { match trace_processor .process_traces(config, req, trace_tx, mini_agent_metadata) .await @@ -156,7 +156,7 @@ impl MiniAgent { ), } } - (&Method::PUT, STATS_ENDPOINT_PATH) => { + (&Method::PUT | &Method::POST, STATS_ENDPOINT_PATH) => { match stats_processor.process_stats(config, req, stats_tx).await { Ok(res) => Ok(res), Err(err) => log_and_create_http_response( diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 62d556d..c2d2134 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -38,6 +38,7 @@ impl StatsProcessor for ServerlessStatsProcessor { req: Request, tx: Sender, ) -> http::Result> { + info!("Recieved trace stats to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( @@ -48,8 +49,6 @@ impl StatsProcessor for ServerlessStatsProcessor { return response; } - info!("Recieved trace stats to process"); - // deserialize trace stats from the request body, convert to protobuf structs (see trace-protobuf crate) let mut stats: pb::ClientStatsPayload = match stats_utils::get_stats_from_request_body(body).await { diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 9ef6268..7da3241 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; -use log::error; +use log::{error, info}; use tokio::sync::mpsc::Sender; use datadog_trace_normalization::normalizer; @@ -55,6 +55,7 @@ impl TraceProcessor for ServerlessTraceProcessor { tx: Sender, mini_agent_metadata: Arc, ) -> http::Result> { + info!("Recieved traces to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( From daeb83998351cfc11ae483f9627f356fdacddcb6 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 19 Jul 2023 08:29:24 -0700 Subject: [PATCH 11/64] Mini Agent: Integrate tag replacement (#195) --- Cargo.toml | 1 + src/config.rs | 20 +++++++++++++++++++- src/trace_processor.rs | 6 ++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ee58ef1..511ac65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ serde_json = "1.0" datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } datadog-trace-normalization = { path = "../trace-normalization" } +datadog-trace-obfuscation = { path = "../trace-obfuscation" } [dev-dependencies] rmp-serde = "1.1.1" diff --git a/src/config.rs b/src/config.rs index bd108fa..c9e06d2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,14 +1,16 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +use log::{debug, error}; use std::env; +use datadog_trace_obfuscation::replacer::{self, ReplaceRule}; use datadog_trace_utils::trace_utils; const TRACE_INTAKE_ROUTE: &str = "/api/v0.2/traces"; const TRACE_STATS_INTAKE_ROUTE: &str = "/api/v0.2/stats"; -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Config { pub api_key: String, pub function_name: Option, @@ -24,6 +26,7 @@ pub struct Config { pub trace_intake_url: String, pub trace_stats_intake_url: String, pub dd_site: String, + pub tag_replace_rules: Option>, } impl Config { @@ -65,6 +68,20 @@ impl Config { trace_stats_intake_url = format!("{endpoint_prefix}{TRACE_STATS_INTAKE_ROUTE}"); }; + let tag_replace_rules: Option> = match env::var("DD_APM_REPLACE_TAGS") { + Ok(replace_rules_str) => match replacer::parse_rules_from_string(&replace_rules_str) { + Ok(res) => { + debug!("Successfully parsed DD_APM_REPLACE_TAGS: {res:?}"); + Some(res) + } + Err(e) => { + error!("Failed to parse DD_APM_REPLACE_TAGS: {e}"); + None + } + }, + Err(_) => None, + }; + Ok(Config { api_key, function_name, @@ -77,6 +94,7 @@ impl Config { dd_site, trace_intake_url, trace_stats_intake_url, + tag_replace_rules, }) } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 7da3241..994422f 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; +use datadog_trace_obfuscation::replacer; use hyper::{http, Body, Request, Response, StatusCode}; use log::{error, info}; use tokio::sync::mpsc::Sender; @@ -121,6 +122,10 @@ impl TraceProcessor for ServerlessTraceProcessor { &config.env_type, ); + if let Some(rules) = &config.tag_replace_rules { + replacer::replace_trace_tags(&mut chunk.spans, rules) + } + trace_chunks.push(chunk); if !gathered_root_span_tags { @@ -258,6 +263,7 @@ mod tests { dd_site: "datadoghq.com".to_string(), env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), + tag_replace_rules: None, } } From a04191c3cd1bf7f2dbc0d756102259d2447de1f5 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Fri, 4 Aug 2023 20:26:35 +0200 Subject: [PATCH 12/64] Implement sending via sidecar (#192) * Implemented tracing and agent sampling in sidecar * Address CR feedback Signed-off-by: Bob Weinand * Polyfill memfd on old glibc targets Signed-off-by: Bob Weinand * Small nit from CR applied Signed-off-by: Bob Weinand --------- Signed-off-by: Bob Weinand --- Cargo.toml | 1 + src/config.rs | 74 ++++++++++----------- src/mini_agent.rs | 5 +- src/stats_flusher.rs | 4 +- src/trace_flusher.rs | 46 +++++-------- src/trace_processor.rs | 142 ++++++++++++++--------------------------- 6 files changed, 103 insertions(+), 169 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 511ac65..91445fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ async-trait = "0.1.64" log = "0.4" serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" +ddcommon = { path = "../ddcommon" } datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } datadog-trace-normalization = { path = "../trace-normalization" } diff --git a/src/config.rs b/src/config.rs index c9e06d2..93b0016 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,18 +1,21 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +use ddcommon::Endpoint; use log::{debug, error}; +use std::borrow::Cow; use std::env; +use std::str::FromStr; use datadog_trace_obfuscation::replacer::{self, ReplaceRule}; +use datadog_trace_utils::config_utils::{ + read_cloud_env, trace_intake_url, trace_intake_url_prefixed, trace_stats_url, + trace_stats_url_prefixed, +}; use datadog_trace_utils::trace_utils; -const TRACE_INTAKE_ROUTE: &str = "/api/v0.2/traces"; -const TRACE_STATS_INTAKE_ROUTE: &str = "/api/v0.2/stats"; - #[derive(Debug)] pub struct Config { - pub api_key: String, pub function_name: Option, pub env_type: trace_utils::EnvironmentType, pub os: String, @@ -23,49 +26,33 @@ pub struct Config { pub stats_flush_interval: u64, /// timeout for environment verification, in milliseconds pub verify_env_timeout: u64, - pub trace_intake_url: String, - pub trace_stats_intake_url: String, + pub trace_intake: Endpoint, + pub trace_stats_intake: Endpoint, pub dd_site: String, pub tag_replace_rules: Option>, } impl Config { pub fn new() -> Result> { - let api_key = env::var("DD_API_KEY") - .map_err(|_| anyhow::anyhow!("DD_API_KEY environment variable is not set"))?; - let mut function_name = None; - - let mut maybe_env_type = None; - if let Ok(res) = env::var("K_SERVICE") { - // Set by Google Cloud Functions for newer runtimes - function_name = Some(res); - maybe_env_type = Some(trace_utils::EnvironmentType::CloudFunction); - } else if let Ok(res) = env::var("FUNCTION_NAME") { - // Set by Google Cloud Functions for older runtimes - function_name = Some(res); - maybe_env_type = Some(trace_utils::EnvironmentType::CloudFunction); - } else if let Ok(res) = env::var("WEBSITE_SITE_NAME") { - // Set by Azure Functions - function_name = Some(res); - maybe_env_type = Some(trace_utils::EnvironmentType::AzureFunction); - } - - let env_type = maybe_env_type.ok_or_else(|| { + let api_key: Cow = env::var("DD_API_KEY") + .map_err(|_| anyhow::anyhow!("DD_API_KEY environment variable is not set"))? + .into(); + + let (function_name, env_type) = read_cloud_env().ok_or_else(|| { anyhow::anyhow!("Unable to identify environment. Shutting down Mini Agent.") })?; let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & trace stats to) - let mut trace_intake_url = format!("https://trace.agent.{dd_site}{TRACE_INTAKE_ROUTE}"); - let mut trace_stats_intake_url = - format!("https://trace.agent.{dd_site}{TRACE_STATS_INTAKE_ROUTE}"); + let mut trace_intake_url = trace_intake_url(&dd_site); + let mut trace_stats_intake_url = trace_stats_url(&dd_site); // DD_APM_DD_URL env var will primarily be used for integration tests // overrides the entire trace/trace stats intake url prefix if let Ok(endpoint_prefix) = env::var("DD_APM_DD_URL") { - trace_intake_url = format!("{endpoint_prefix}{TRACE_INTAKE_ROUTE}"); - trace_stats_intake_url = format!("{endpoint_prefix}{TRACE_STATS_INTAKE_ROUTE}"); + trace_intake_url = trace_intake_url_prefixed(&endpoint_prefix); + trace_stats_intake_url = trace_stats_url_prefixed(&endpoint_prefix); }; let tag_replace_rules: Option> = match env::var("DD_APM_REPLACE_TAGS") { @@ -83,8 +70,7 @@ impl Config { }; Ok(Config { - api_key, - function_name, + function_name: Some(function_name), env_type, os: env::consts::OS.to_string(), max_request_content_length: 10 * 1024 * 1024, // 10MB in Bytes @@ -92,8 +78,14 @@ impl Config { stats_flush_interval: 3, verify_env_timeout: 100, dd_site, - trace_intake_url, - trace_stats_intake_url, + trace_intake: Endpoint { + url: hyper::Uri::from_str(&trace_intake_url).unwrap(), + api_key: Some(api_key.clone()), + }, + trace_stats_intake: Endpoint { + url: hyper::Uri::from_str(&trace_stats_intake_url).unwrap(), + api_key: Some(api_key), + }, tag_replace_rules, }) } @@ -141,11 +133,11 @@ mod tests { assert!(config_res.is_ok()); let config = config_res.unwrap(); assert_eq!( - config.trace_intake_url, + config.trace_intake.url, "https://trace.agent.datadoghq.com/api/v0.2/traces" ); assert_eq!( - config.trace_stats_intake_url, + config.trace_stats_intake.url, "https://trace.agent.datadoghq.com/api/v0.2/stats" ); env::remove_var("DD_API_KEY"); @@ -170,7 +162,7 @@ mod tests { let config_res = config::Config::new(); assert!(config_res.is_ok()); let config = config_res.unwrap(); - assert_eq!(config.trace_intake_url, expected_url); + assert_eq!(config.trace_intake.url, expected_url); env::remove_var("DD_API_KEY"); env::remove_var("DD_SITE"); env::remove_var("K_SERVICE"); @@ -194,7 +186,7 @@ mod tests { let config_res = config::Config::new(); assert!(config_res.is_ok()); let config = config_res.unwrap(); - assert_eq!(config.trace_stats_intake_url, expected_url); + assert_eq!(config.trace_stats_intake.url, expected_url); env::remove_var("DD_API_KEY"); env::remove_var("DD_SITE"); env::remove_var("K_SERVICE"); @@ -210,11 +202,11 @@ mod tests { assert!(config_res.is_ok()); let config = config_res.unwrap(); assert_eq!( - config.trace_intake_url, + config.trace_intake.url, "http://127.0.0.1:3333/api/v0.2/traces" ); assert_eq!( - config.trace_stats_intake_url, + config.trace_stats_intake.url, "http://127.0.0.1:3333/api/v0.2/stats" ); env::remove_var("DD_API_KEY"); diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 774297e..d5f231d 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -15,6 +15,7 @@ use crate::http_utils::log_and_create_http_response; use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; +use datadog_trace_utils::trace_utils::SendData; const MINI_AGENT_PORT: usize = 8126; const TRACE_ENDPOINT_PATH: &str = "/v0.4/traces"; @@ -55,7 +56,7 @@ impl MiniAgent { // setup a channel to send processed traces to our flusher. tx is passed through each endpoint_handler // to the trace processor, which uses it to send de-serialized processed trace payloads to our trace flusher. - let (trace_tx, trace_rx): (Sender, Receiver) = + let (trace_tx, trace_rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); // start our trace flusher. receives trace payloads and handles buffering + deciding when to flush to backend. @@ -138,7 +139,7 @@ impl MiniAgent { config: Arc, req: Request, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, stats_processor: Arc, stats_tx: Sender, mini_agent_metadata: Arc, diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index badda1e..dd98074 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -77,8 +77,8 @@ impl StatsFlusher for ServerlessStatsFlusher { match stats_utils::send_stats_payload( serialized_stats_payload, - &config.trace_stats_intake_url, - &config.api_key, + &config.trace_stats_intake, + config.trace_stats_intake.api_key.as_ref().unwrap(), ) .await { diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index 8f59a72..9078988 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -2,12 +2,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. use async_trait::async_trait; -use log::{debug, error, info}; +use log::{error, info}; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; -use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; +use datadog_trace_utils::trace_utils::SendData; use crate::config::Config; @@ -15,9 +15,9 @@ use crate::config::Config; pub trait TraceFlusher { /// Starts a trace flusher that listens for trace payloads sent to the tokio mpsc Receiver, /// implementing flushing logic that calls flush_traces. - async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver); + async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver); /// Flushes traces to the Datadog trace intake. - async fn flush_traces(&self, config: Arc, traces: Vec); + async fn flush_traces(&self, traces: Vec); } #[derive(Clone)] @@ -25,8 +25,8 @@ pub struct ServerlessTraceFlusher {} #[async_trait] impl TraceFlusher for ServerlessTraceFlusher { - async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver) { - let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); + async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver) { + let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); let buffer_producer = buffer.clone(); let buffer_consumer = buffer.clone(); @@ -43,41 +43,25 @@ impl TraceFlusher for ServerlessTraceFlusher { let mut buffer = buffer_consumer.lock().await; if !buffer.is_empty() { - self.flush_traces(config.clone(), buffer.to_vec()).await; + self.flush_traces(buffer.to_vec()).await; buffer.clear(); } } } - async fn flush_traces(&self, config: Arc, traces: Vec) { + async fn flush_traces(&self, traces: Vec) { if traces.is_empty() { return; } info!("Flushing {} traces", traces.len()); - let agent_payload = trace_utils::construct_agent_payload(traces); - - debug!("Trace agent payload to be sent: {agent_payload:?}"); - - let serialized_agent_payload = match trace_utils::serialize_agent_payload(agent_payload) { - Ok(res) => res, - Err(err) => { - error!("Failed to serialize trace agent payload, dropping traces: {err}"); - return; - } - }; - - match trace_utils::send( - serialized_agent_payload, - &config.trace_intake_url, - &config.api_key, - ) - .await - { - Ok(_) => info!("Successfully flushed traces"), - Err(e) => { - error!("Error sending trace: {e:?}") - // TODO: Retries + for traces in trace_utils::coalesce_send_data(traces) { + match traces.send().await { + Ok(_) => info!("Successfully flushed traces"), + Err(e) => { + error!("Error sending trace: {e:?}") + // TODO: Retries + } } } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 994422f..40797f4 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -4,34 +4,19 @@ use std::sync::Arc; use async_trait::async_trait; -use datadog_trace_obfuscation::replacer; use hyper::{http, Body, Request, Response, StatusCode}; -use log::{error, info}; +use log::info; use tokio::sync::mpsc::Sender; -use datadog_trace_normalization::normalizer; -use datadog_trace_protobuf::pb; +use datadog_trace_obfuscation::replacer; use datadog_trace_utils::trace_utils; +use datadog_trace_utils::trace_utils::SendData; use crate::{ config::Config, http_utils::{self, log_and_create_http_response}, }; -/// Used to populate root_span_tags fields if they exist in the root span's meta tags -macro_rules! parse_root_span_tags { - ( - $root_span_meta_map:ident, - { $($tag:literal => $($root_span_tags_struct_field:ident).+ ,)+ } - ) => { - $( - if let Some(root_span_tag_value) = $root_span_meta_map.get($tag) { - $($root_span_tags_struct_field).+ = root_span_tag_value; - } - )+ - } -} - #[async_trait] pub trait TraceProcessor { /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc Sender. @@ -39,7 +24,7 @@ pub trait TraceProcessor { &self, config: Arc, req: Request, - tx: Sender, + tx: Sender, mini_agent_metadata: Arc, ) -> http::Result>; } @@ -53,7 +38,7 @@ impl TraceProcessor for ServerlessTraceProcessor { &self, config: Arc, req: Request, - tx: Sender, + tx: Sender, mini_agent_metadata: Arc, ) -> http::Result> { info!("Recieved traces to process"); @@ -67,10 +52,10 @@ impl TraceProcessor for ServerlessTraceProcessor { return response; } - let tracer_header_tags = trace_utils::get_tracer_header_tags(&parts.headers); + let tracer_header_tags = (&parts.headers).into(); // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf crate) - let mut traces = match trace_utils::get_traces_from_request_body(body).await { + let (body_size, traces) = match trace_utils::get_traces_from_request_body(body).await { Ok(res) => res, Err(err) => { return log_and_create_http_response( @@ -80,74 +65,29 @@ impl TraceProcessor for ServerlessTraceProcessor { } }; - let mut trace_chunks: Vec = Vec::new(); - - let mut gathered_root_span_tags = false; - let mut root_span_tags = trace_utils::RootSpanTags::default(); - - for trace in traces.iter_mut() { - if let Err(e) = normalizer::normalize_trace(trace) { - error!("Error normalizing trace: {e}"); - } - - let mut chunk = trace_utils::construct_trace_chunk(trace.to_vec()); - - let root_span_index = match trace_utils::get_root_span_index(trace) { - Ok(res) => res, - Err(e) => { - error!("Error getting the root span index of a trace, skipping. {e}"); - continue; + let payload = trace_utils::collect_trace_chunks( + traces, + &tracer_header_tags, + |chunk, root_span_index| { + trace_utils::set_serverless_root_span_tags( + &mut chunk.spans[root_span_index], + config.function_name.clone(), + &config.env_type, + ); + for span in chunk.spans.iter_mut() { + trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); } - }; - if let Err(e) = normalizer::normalize_chunk(&mut chunk, root_span_index) { - error!("Error normalizing trace chunk: {e}"); - } - - for span in chunk.spans.iter_mut() { - // TODO: obfuscate & truncate spans - if tracer_header_tags.client_computed_top_level { - trace_utils::update_tracer_top_level(span); + if let Some(rules) = &config.tag_replace_rules { + replacer::replace_trace_tags(&mut chunk.spans, rules) } - trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - } - - if !tracer_header_tags.client_computed_top_level { - trace_utils::compute_top_level_span(&mut chunk.spans); - } - - trace_utils::set_serverless_root_span_tags( - &mut chunk.spans[root_span_index], - config.function_name.clone(), - &config.env_type, - ); - - if let Some(rules) = &config.tag_replace_rules { - replacer::replace_trace_tags(&mut chunk.spans, rules) - } - - trace_chunks.push(chunk); - - if !gathered_root_span_tags { - gathered_root_span_tags = true; - let meta_map = &trace[root_span_index].meta; - parse_root_span_tags!( - meta_map, - { - "env" => root_span_tags.env, - "version" => root_span_tags.app_version, - "_dd.hostname" => root_span_tags.hostname, - "runtime-id" => root_span_tags.runtime_id, - } - ); - } - } + }, + ); - let tracer_payload = - trace_utils::construct_tracer_payload(trace_chunks, tracer_header_tags, root_span_tags); + let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); // send trace payload to our trace flusher - match tx.send(tracer_payload).await { + match tx.send(send_data).await { Ok(_) => { return log_and_create_http_response( "Successfully buffered traces to be flushed.", @@ -181,6 +121,7 @@ mod tests { }; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils; + use ddcommon::Endpoint; fn create_test_span(start: i64, span_id: u64, parent_id: u64, is_top_level: bool) -> pb::Span { let mut span = pb::Span { @@ -252,14 +193,19 @@ mod tests { fn create_test_config() -> Config { Config { - api_key: "dummy_api_key".to_string(), function_name: Some("dummy_function_name".to_string()), max_request_content_length: 10 * 1024 * 1024, trace_flush_interval: 3, stats_flush_interval: 3, verify_env_timeout: 100, - trace_intake_url: "trace.agent.notdog.com/traces".to_string(), - trace_stats_intake_url: "trace.agent.notdog.com/stats".to_string(), + trace_intake: Endpoint { + url: hyper::Uri::from_static("https://trace.agent.notdog.com/traces"), + api_key: Some("dummy_api_key".into()), + }, + trace_stats_intake: Endpoint { + url: hyper::Uri::from_static("https://trace.agent.notdog.com/stats"), + api_key: Some("dummy_api_key".into()), + }, dd_site: "datadoghq.com".to_string(), env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), @@ -269,8 +215,10 @@ mod tests { #[tokio::test] async fn test_process_trace() { - let (tx, mut rx): (Sender, Receiver) = - mpsc::channel(1); + let (tx, mut rx): ( + Sender, + Receiver, + ) = mpsc::channel(1); let start = get_current_timestamp_nanos(); @@ -321,13 +269,18 @@ mod tests { app_version: "".to_string(), }; - assert_eq!(expected_tracer_payload, tracer_payload.unwrap()); + assert_eq!( + expected_tracer_payload, + tracer_payload.unwrap().get_payloads()[0] + ); } #[tokio::test] async fn test_process_trace_top_level_span_set() { - let (tx, mut rx): (Sender, Receiver) = - mpsc::channel(1); + let (tx, mut rx): ( + Sender, + Receiver, + ) = mpsc::channel(1); let start = get_current_timestamp_nanos(); @@ -386,6 +339,9 @@ mod tests { app_version: "".to_string(), }; - assert_eq!(expected_tracer_payload, tracer_payload.unwrap()); + assert_eq!( + expected_tracer_payload, + tracer_payload.unwrap().get_payloads()[0] + ); } } From 40398c4dfa3e7bcfa1c13f5ead8eb2c823c9d31b Mon Sep 17 00:00:00 2001 From: David Lee Date: Fri, 25 Aug 2023 10:08:39 -0700 Subject: [PATCH 13/64] Refactor common unit test create_test_span function into trace-utils (#226) --- src/trace_processor.rs | 88 +++++++----------------------------------- 1 file changed, 14 insertions(+), 74 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 40797f4..bfd789e 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -107,7 +107,6 @@ impl TraceProcessor for ServerlessTraceProcessor { #[cfg(test)] mod tests { use hyper::Request; - use serde_json::json; use std::{ collections::HashMap, sync::Arc, @@ -120,70 +119,12 @@ mod tests { trace_processor::{self, TraceProcessor}, }; use datadog_trace_protobuf::pb; - use datadog_trace_utils::trace_utils; + use datadog_trace_utils::{ + trace_test_utils::{create_test_json_span, create_test_span}, + trace_utils, + }; use ddcommon::Endpoint; - fn create_test_span(start: i64, span_id: u64, parent_id: u64, is_top_level: bool) -> pb::Span { - let mut span = pb::Span { - trace_id: 111, - span_id, - service: "test-service".to_string(), - name: "test_name".to_string(), - resource: "test-resource".to_string(), - parent_id, - start, - duration: 5, - error: 0, - meta: HashMap::from([ - ("service".to_string(), "test-service".to_string()), - ("env".to_string(), "test-env".to_string()), - ( - "runtime-id".to_string(), - "afjksdljfkllksdj-28934889".to_string(), - ), - ]), - metrics: HashMap::new(), - r#type: "".to_string(), - meta_struct: HashMap::new(), - }; - if is_top_level { - span.metrics.insert("_top_level".to_string(), 1.0); - span.meta - .insert("_dd.origin".to_string(), "cloudfunction".to_string()); - span.meta - .insert("origin".to_string(), "cloudfunction".to_string()); - span.meta.insert( - "functionname".to_string(), - "dummy_function_name".to_string(), - ); - span.r#type = "serverless".to_string(); - } - span - } - - fn create_test_json_span(start: i64, span_id: u64, parent_id: u64) -> serde_json::Value { - json!( - { - "trace_id": 111, - "span_id": span_id, - "service": "test-service", - "name": "test_name", - "resource": "test-resource", - "parent_id": parent_id, - "start": start, - "duration": 5, - "error": 0, - "meta": { - "service": "test-service", - "env": "test-env", - "runtime-id": "afjksdljfkllksdj-28934889", - }, - "metrics": {}, - "meta_struct": {}, - } - ) - } - fn get_current_timestamp_nanos() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -222,7 +163,7 @@ mod tests { let start = get_current_timestamp_nanos(); - let json_span = create_test_json_span(start, 222, 0); + let json_span = create_test_json_span(11, 222, 333, start); let bytes = rmp_serde::to_vec(&vec![vec![json_span]]).unwrap(); let request = Request::builder() @@ -255,11 +196,11 @@ mod tests { language_name: "nodejs".to_string(), language_version: "v19.7.0".to_string(), tracer_version: "4.0.0".to_string(), - runtime_id: "afjksdljfkllksdj-28934889".to_string(), + runtime_id: "test-runtime-id-value".to_string(), chunks: vec![pb::TraceChunk { priority: i8::MIN as i32, origin: "".to_string(), - spans: vec![create_test_span(start, 222, 0, true)], + spans: vec![create_test_span(11, 222, 333, start, true)], tags: HashMap::new(), dropped_trace: false, }], @@ -285,9 +226,9 @@ mod tests { let start = get_current_timestamp_nanos(); let json_trace = vec![ - create_test_json_span(start, 333, 222), - create_test_json_span(start, 222, 0), - create_test_json_span(start, 444, 333), + create_test_json_span(11, 333, 222, start), + create_test_json_span(11, 222, 0, start), + create_test_json_span(11, 444, 333, start), ]; let bytes = rmp_serde::to_vec(&vec![json_trace]).unwrap(); @@ -321,14 +262,14 @@ mod tests { language_name: "nodejs".to_string(), language_version: "v19.7.0".to_string(), tracer_version: "4.0.0".to_string(), - runtime_id: "afjksdljfkllksdj-28934889".to_string(), + runtime_id: "test-runtime-id-value".to_string(), chunks: vec![pb::TraceChunk { priority: i8::MIN as i32, origin: "".to_string(), spans: vec![ - create_test_span(start, 333, 222, false), - create_test_span(start, 222, 0, true), - create_test_span(start, 444, 333, false), + create_test_span(11, 333, 222, start, false), + create_test_span(11, 222, 0, start, true), + create_test_span(11, 444, 333, start, false), ], tags: HashMap::new(), dropped_trace: false, @@ -338,7 +279,6 @@ mod tests { hostname: "".to_string(), app_version: "".to_string(), }; - assert_eq!( expected_tracer_payload, tracer_payload.unwrap().get_payloads()[0] From b056a8e68873d4c05770d24e87929ca918edab29 Mon Sep 17 00:00:00 2001 From: David Lee Date: Fri, 8 Sep 2023 10:04:16 -0700 Subject: [PATCH 14/64] Http url string obfuscation (#228) --- src/config.rs | 25 ++++++++----------------- src/trace_processor.rs | 10 ++++------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/config.rs b/src/config.rs index 93b0016..1702655 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,12 +2,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. use ddcommon::Endpoint; -use log::{debug, error}; use std::borrow::Cow; use std::env; use std::str::FromStr; -use datadog_trace_obfuscation::replacer::{self, ReplaceRule}; +use datadog_trace_obfuscation::obfuscation_config; use datadog_trace_utils::config_utils::{ read_cloud_env, trace_intake_url, trace_intake_url_prefixed, trace_stats_url, trace_stats_url_prefixed, @@ -29,7 +28,7 @@ pub struct Config { pub trace_intake: Endpoint, pub trace_stats_intake: Endpoint, pub dd_site: String, - pub tag_replace_rules: Option>, + pub obfuscation_config: obfuscation_config::ObfuscationConfig, } impl Config { @@ -55,19 +54,11 @@ impl Config { trace_stats_intake_url = trace_stats_url_prefixed(&endpoint_prefix); }; - let tag_replace_rules: Option> = match env::var("DD_APM_REPLACE_TAGS") { - Ok(replace_rules_str) => match replacer::parse_rules_from_string(&replace_rules_str) { - Ok(res) => { - debug!("Successfully parsed DD_APM_REPLACE_TAGS: {res:?}"); - Some(res) - } - Err(e) => { - error!("Failed to parse DD_APM_REPLACE_TAGS: {e}"); - None - } - }, - Err(_) => None, - }; + let obfuscation_config = obfuscation_config::ObfuscationConfig::new().map_err(|err| { + anyhow::anyhow!( + "Error creating obfuscation config, Mini Agent will not start. Error: {err}", + ) + })?; Ok(Config { function_name: Some(function_name), @@ -86,7 +77,7 @@ impl Config { url: hyper::Uri::from_str(&trace_stats_intake_url).unwrap(), api_key: Some(api_key), }, - tag_replace_rules, + obfuscation_config, }) } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index bfd789e..2863d22 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -8,7 +8,7 @@ use hyper::{http, Body, Request, Response, StatusCode}; use log::info; use tokio::sync::mpsc::Sender; -use datadog_trace_obfuscation::replacer; +use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; @@ -76,10 +76,7 @@ impl TraceProcessor for ServerlessTraceProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - } - - if let Some(rules) = &config.tag_replace_rules { - replacer::replace_trace_tags(&mut chunk.spans, rules) + obfuscate_span(span, &config.obfuscation_config); } }, ); @@ -106,6 +103,7 @@ impl TraceProcessor for ServerlessTraceProcessor { #[cfg(test)] mod tests { + use datadog_trace_obfuscation::obfuscation_config::ObfuscationConfig; use hyper::Request; use std::{ collections::HashMap, @@ -150,7 +148,7 @@ mod tests { dd_site: "datadoghq.com".to_string(), env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), - tag_replace_rules: None, + obfuscation_config: ObfuscationConfig::new().unwrap(), } } From dee5f9fb7a35419bcd4d52207a2bfa785ff523b6 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Fri, 6 Oct 2023 11:38:23 -0400 Subject: [PATCH 15/64] Run Miri everywhere (#269) * Explicitly mark all tests where MIRI doesn't run * Update the github action to run miri on the entire library --- src/env_verifier.rs | 9 +++++++++ src/http_utils.rs | 4 ++++ src/trace_processor.rs | 2 ++ 3 files changed, 15 insertions(+) diff --git a/src/env_verifier.rs b/src/env_verifier.rs index d8f6aa4..1d08604 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -299,6 +299,7 @@ mod tests { use super::{EnvVerifier, ServerlessEnvVerifier}; #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_gcp_env_false_if_metadata_server_unreachable() { struct MockGoogleMetadataClient {} #[async_trait] @@ -316,6 +317,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_gcp_env_false_if_no_server_in_response_headers() { struct MockGoogleMetadataClient {} #[async_trait] @@ -336,6 +338,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_gcp_env_if_server_header_not_serverless() { struct MockGoogleMetadataClient {} #[async_trait] @@ -357,6 +360,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_gcp_env_true_if_cloud_function_env() { struct MockGoogleMetadataClient {} #[async_trait] @@ -395,6 +399,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_gcp_verify_environment_timeout_exceeded_gives_unknown_values() { let env_verifier = ServerlessEnvVerifier {}; let res = env_verifier @@ -428,6 +433,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_azure_env_windows_true() { struct MockAzureVerificationClient {} #[async_trait] @@ -443,6 +449,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_azure_env_windows_false() { struct MockAzureVerificationClient {} #[async_trait] @@ -465,6 +472,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_azure_env_linux_true() { struct MockAzureVerificationClient {} #[async_trait] @@ -480,6 +488,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_ensure_azure_env_linux_false() { struct MockAzureVerificationClient {} #[async_trait] diff --git a/src/http_utils.rs b/src/http_utils.rs index f826134..86c5983 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -100,6 +100,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_request_content_length_missing() { let verify_result = verify_request_content_length(&HeaderMap::new(), 1, "Test Prefix"); assert!(verify_result.is_some()); @@ -113,6 +114,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_request_content_length_cant_convert_to_str() { let verify_result = verify_request_content_length( &create_test_headers_with_content_length("❤❤❤❤❤❤❤"), @@ -130,6 +132,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_request_content_length_cant_convert_to_usize() { let verify_result = verify_request_content_length( &create_test_headers_with_content_length("not_an_int"), @@ -147,6 +150,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_request_content_length_too_long() { let verify_result = verify_request_content_length( &create_test_headers_with_content_length("100"), diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 2863d22..08a5bf4 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -153,6 +153,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_process_trace() { let (tx, mut rx): ( Sender, @@ -215,6 +216,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn test_process_trace_top_level_span_set() { let (tx, mut rx): ( Sender, From 0f7ded5ee2a52fe7027b1530bbb32d981e7cb6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 5 Mar 2024 14:34:01 +0100 Subject: [PATCH 16/64] Reformat License and add SPDX headers --- src/config.rs | 4 ++-- src/env_verifier.rs | 5 +++-- src/http_utils.rs | 4 ++-- src/lib.rs | 4 ++-- src/mini_agent.rs | 4 ++-- src/stats_flusher.rs | 4 ++-- src/stats_processor.rs | 4 ++-- src/trace_flusher.rs | 4 ++-- src/trace_processor.rs | 4 ++-- 9 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/config.rs b/src/config.rs index 1702655..1fb2a4c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use ddcommon::Endpoint; use std::borrow::Cow; diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 1d08604..5b5ac06 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -1,5 +1,6 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + use async_trait::async_trait; use hyper::{Body, Client, Method, Request, Response}; use log::{debug, error}; diff --git a/src/http_utils.rs b/src/http_utils.rs index 86c5983..17aa876 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use hyper::{ header, diff --git a/src/lib.rs b/src/lib.rs index 3c9534f..ccbff24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 pub mod config; pub mod env_verifier; diff --git a/src/mini_agent.rs b/src/mini_agent.rs index d5f231d..411c64c 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use hyper::service::{make_service_fn, service_fn}; use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index dd98074..9d7bed7 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; use log::{debug, error, info}; diff --git a/src/stats_processor.rs b/src/stats_processor.rs index c2d2134..23e7937 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index 9078988..c872a02 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; use log::{error, info}; diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 08a5bf4..89f6957 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use std::sync::Arc; From 97d7e94be10073d8c2193feb0235d5c66b746523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 5 Mar 2024 14:34:05 +0100 Subject: [PATCH 17/64] Add comment wrapping and lenghts to rustfmt --- src/config.rs | 3 ++- src/env_verifier.rs | 6 ++++-- src/http_utils.rs | 11 ++++++----- src/mini_agent.rs | 8 +++++--- src/stats_processor.rs | 3 ++- src/trace_processor.rs | 6 ++++-- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/config.rs b/src/config.rs index 1fb2a4c..f88931a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -43,7 +43,8 @@ impl Config { let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); - // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & trace stats to) + // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & + // trace stats to) let mut trace_intake_url = trace_intake_url(&dd_site); let mut trace_stats_intake_url = trace_stats_url(&dd_site); diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 5b5ac06..56cb941 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -129,7 +129,8 @@ fn get_region_from_gcp_region_string(str: String) -> String { } } -/// GoogleMetadataClient trait is used so we can mock a google metadata server response in unit tests +/// GoogleMetadataClient trait is used so we can mock a google metadata server response in unit +/// tests #[async_trait] trait GoogleMetadataClient { async fn get_metadata(&self) -> anyhow::Result>; @@ -212,7 +213,8 @@ async fn verify_azure_environment_or_exit(os: &str) { ); } -/// AzureVerificationClient trait is used so we can mock the azure function local url response in unit tests +/// AzureVerificationClient trait is used so we can mock the azure function local url response in +/// unit tests trait AzureVerificationClient { fn get_function_root_files(&self, path: &Path) -> anyhow::Result>; } diff --git a/src/http_utils.rs b/src/http_utils.rs index 17aa876..701af8d 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -10,7 +10,8 @@ use log::{error, info}; use serde_json::json; /// Does two things: -/// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be written, +/// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be +/// written, /// otherwise error will be written. /// 2. Returns the given message in the body of JSON response with the given status code. /// @@ -31,11 +32,11 @@ pub fn log_and_create_http_response( Response::builder().status(status).body(Body::from(body)) } -/// Takes a request's header map, and verifies that the "content-length" header is present, valid, and less -/// than the given max_content_length. +/// Takes a request's header map, and verifies that the "content-length" header is present, valid, +/// and less than the given max_content_length. /// -/// Will return None if no issues are found. Otherwise logs an error (with the given prefix) and returns -/// and HTTP Response with the appropriate error status code. +/// Will return None if no issues are found. Otherwise logs an error (with the given prefix) and +/// returns and HTTP Response with the appropriate error status code. pub fn verify_request_content_length( header_map: &HeaderMap, max_content_length: usize, diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 411c64c..e42103b 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -54,12 +54,14 @@ impl MiniAgent { now.elapsed().as_millis() ); - // setup a channel to send processed traces to our flusher. tx is passed through each endpoint_handler - // to the trace processor, which uses it to send de-serialized processed trace payloads to our trace flusher. + // setup a channel to send processed traces to our flusher. tx is passed through each + // endpoint_handler to the trace processor, which uses it to send de-serialized + // processed trace payloads to our trace flusher. let (trace_tx, trace_rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); - // start our trace flusher. receives trace payloads and handles buffering + deciding when to flush to backend. + // start our trace flusher. receives trace payloads and handles buffering + deciding when to + // flush to backend. let trace_flusher = self.trace_flusher.clone(); let trace_config = self.config.clone(); tokio::spawn(async move { diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 23e7937..5d9ee5d 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -49,7 +49,8 @@ impl StatsProcessor for ServerlessStatsProcessor { return response; } - // deserialize trace stats from the request body, convert to protobuf structs (see trace-protobuf crate) + // deserialize trace stats from the request body, convert to protobuf structs (see + // trace-protobuf crate) let mut stats: pb::ClientStatsPayload = match stats_utils::get_stats_from_request_body(body).await { Ok(res) => res, diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 89f6957..e9a4866 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -19,7 +19,8 @@ use crate::{ #[async_trait] pub trait TraceProcessor { - /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc Sender. + /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc + /// Sender. async fn process_traces( &self, config: Arc, @@ -54,7 +55,8 @@ impl TraceProcessor for ServerlessTraceProcessor { let tracer_header_tags = (&parts.headers).into(); - // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf crate) + // deserialize traces from the request body, convert to protobuf structs (see trace-protobuf + // crate) let (body_size, traces) = match trace_utils::get_traces_from_request_body(body).await { Ok(res) => res, Err(err) => { From 9c53e5b614aa709a12175f4f0d4177672530e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 16 Apr 2024 17:32:19 +0200 Subject: [PATCH 18/64] Make verify timeout test in datadog-trace-mini-agent trigger timeout * Only setting the timeout to 0 ms is not enough, and there was a race in the test between the dns lookup and the timeout that intermittently failed the test (most likely on Windows). --- src/env_verifier.rs | 142 ++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 46 deletions(-) diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 56cb941..714456d 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use std::fs; use std::path::Path; use std::process; +use std::sync::Arc; use std::time::{Duration, Instant}; use datadog_trace_utils::trace_utils; @@ -61,7 +62,64 @@ pub trait EnvVerifier { ) -> trace_utils::MiniAgentMetadata; } -pub struct ServerlessEnvVerifier {} +pub struct ServerlessEnvVerifier { + gmc: Arc>, +} + +impl Default for ServerlessEnvVerifier { + fn default() -> Self { + Self::new() + } +} + +impl ServerlessEnvVerifier { + pub fn new() -> Self { + Self { + gmc: Arc::new(Box::new(GoogleMetadataClientWrapper {})), + } + } + + #[cfg(test)] + pub(crate) fn new_with_google_metadata_client( + gmc: Box, + ) -> Self { + Self { gmc: Arc::new(gmc) } + } + + async fn verify_gcp_environment_or_exit( + &self, + verify_env_timeout: u64, + ) -> trace_utils::MiniAgentMetadata { + let gcp_metadata_request = ensure_gcp_function_environment(self.gmc.as_ref().as_ref()); + let gcp_metadata = match tokio::time::timeout( + Duration::from_millis(verify_env_timeout), + gcp_metadata_request, + ) + .await + { + Ok(result) => match result { + Ok(metadata) => { + debug!("Successfully fetched Google Metadata."); + metadata + } + Err(err) => { + error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {err}"); + process::exit(1); + } + }, + Err(_) => { + error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); + GCPMetadata::default() + } + }; + trace_utils::MiniAgentMetadata { + gcp_project_id: Some(gcp_metadata.project.project_id), + gcp_region: Some(get_region_from_gcp_region_string( + gcp_metadata.instance.region, + )), + } + } +} #[async_trait] impl EnvVerifier for ServerlessEnvVerifier { @@ -77,44 +135,14 @@ impl EnvVerifier for ServerlessEnvVerifier { trace_utils::MiniAgentMetadata::default() } trace_utils::EnvironmentType::CloudFunction => { - return verify_gcp_environment_or_exit(verify_env_timeout).await; + return self + .verify_gcp_environment_or_exit(verify_env_timeout) + .await; } } } } -async fn verify_gcp_environment_or_exit(verify_env_timeout: u64) -> trace_utils::MiniAgentMetadata { - let gcp_metadata_request = - ensure_gcp_function_environment(Box::new(GoogleMetadataClientWrapper {})); - let gcp_metadata = match tokio::time::timeout( - Duration::from_millis(verify_env_timeout), - gcp_metadata_request, - ) - .await - { - Ok(result) => match result { - Ok(metadata) => { - debug!("Successfully fetched Google Metadata."); - metadata - } - Err(err) => { - error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {err}"); - process::exit(1); - } - }, - Err(_) => { - error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); - GCPMetadata::default() - } - }; - trace_utils::MiniAgentMetadata { - gcp_project_id: Some(gcp_metadata.project.project_id), - gcp_region: Some(get_region_from_gcp_region_string( - gcp_metadata.instance.region, - )), - } -} - /// The region found in GCP Metadata comes in the format: "projects/123123/regions/us-east1" /// This function extracts just the region (us-east1) from this GCP region string. /// If the string does not have 4 parts (separated by "/") or extraction fails, return "unknown" @@ -132,7 +160,7 @@ fn get_region_from_gcp_region_string(str: String) -> String { /// GoogleMetadataClient trait is used so we can mock a google metadata server response in unit /// tests #[async_trait] -trait GoogleMetadataClient { +pub(crate) trait GoogleMetadataClient { async fn get_metadata(&self) -> anyhow::Result>; } struct GoogleMetadataClientWrapper {} @@ -159,7 +187,7 @@ impl GoogleMetadataClient for GoogleMetadataClientWrapper { /// If true, returns Metadata from the Google Cloud environment. /// Otherwise, returns an error with the verification failure reason. async fn ensure_gcp_function_environment( - metadata_client: Box, + metadata_client: &(dyn GoogleMetadataClient + Send + Sync), ) -> anyhow::Result { let response = metadata_client.get_metadata().await.map_err(|err| { anyhow::anyhow!("Can't communicate with Google Metadata Server. Error: {err}") @@ -290,7 +318,7 @@ mod tests { use hyper::{Body, Response, StatusCode}; use serde_json::json; use serial_test::serial; - use std::{fs, path::Path}; + use std::{fs, path::Path, time::Duration}; use crate::env_verifier::{ ensure_azure_function_environment, ensure_gcp_function_environment, @@ -311,7 +339,9 @@ mod tests { anyhow::bail!("Random Error") } } - let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + let gmc = + Box::new(MockGoogleMetadataClient {}) as Box; + let res = ensure_gcp_function_environment(gmc.as_ref()).await; assert!(res.is_err()); assert_eq!( res.unwrap_err().to_string(), @@ -332,7 +362,9 @@ mod tests { .unwrap()) } } - let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + let gmc = + Box::new(MockGoogleMetadataClient {}) as Box; + let res = ensure_gcp_function_environment(gmc.as_ref()).await; assert!(res.is_err()); assert_eq!( res.unwrap_err().to_string(), @@ -354,7 +386,9 @@ mod tests { .unwrap()) } } - let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + let gmc = + Box::new(MockGoogleMetadataClient {}) as Box; + let res = ensure_gcp_function_environment(gmc.as_ref()).await; assert!(res.is_err()); assert_eq!( res.unwrap_err().to_string(), @@ -386,7 +420,9 @@ mod tests { .unwrap()) } } - let res = ensure_gcp_function_environment(Box::new(MockGoogleMetadataClient {})).await; + let gmc = + Box::new(MockGoogleMetadataClient {}) as Box; + let res = ensure_gcp_function_environment(gmc.as_ref()).await; assert!(res.is_ok()); assert_eq!( res.unwrap(), @@ -398,23 +434,37 @@ mod tests { project_id: "my-project".to_string() } } - ) + ); } #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_gcp_verify_environment_timeout_exceeded_gives_unknown_values() { - let env_verifier = ServerlessEnvVerifier {}; + struct MockGoogleMetadataClient {} + #[async_trait] + impl GoogleMetadataClient for MockGoogleMetadataClient { + async fn get_metadata(&self) -> anyhow::Result> { + // Sleep for 5 seconds to let the timeout trigger + tokio::time::sleep(Duration::from_secs(5)).await; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::empty()) + .unwrap()) + } + } + let gmc = + Box::new(MockGoogleMetadataClient {}) as Box; + let env_verifier = ServerlessEnvVerifier::new_with_google_metadata_client(gmc); let res = env_verifier - .verify_environment(0, &trace_utils::EnvironmentType::CloudFunction, "linux") - .await; // set the verify_env_timeout to timeout immediately + .verify_environment(100, &trace_utils::EnvironmentType::CloudFunction, "linux") + .await; // set the verify_env_timeout to a small value to trigger the timeout assert_eq!( res, trace_utils::MiniAgentMetadata { gcp_project_id: Some("unknown".to_string()), gcp_region: Some("unknown".to_string()), } - ) + ); } #[test] From b2d6599d6c65d834f70822f90d3d09e6ed026419 Mon Sep 17 00:00:00 2001 From: Pierre Bonet Date: Wed, 15 May 2024 15:31:14 +0200 Subject: [PATCH 19/64] Add descriptions to cargo.toml files, mainly to help discover the repo --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 91445fc..5596879 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "datadog-trace-mini-agent" +description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (GCF and Azure Consumptions plan)" version = "0.1.0" edition = "2021" From 57979cc8f3af40baaa795fad690b8f89b191488f Mon Sep 17 00:00:00 2001 From: Luc Vieillescazes Date: Tue, 7 May 2024 10:08:58 +0200 Subject: [PATCH 20/64] Collect some trace_api.* metrics in the trace flusher --- src/trace_flusher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index c872a02..ba28091 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -56,7 +56,7 @@ impl TraceFlusher for ServerlessTraceFlusher { info!("Flushing {} traces", traces.len()); for traces in trace_utils::coalesce_send_data(traces) { - match traces.send().await { + match traces.send().await.last_result { Ok(_) => info!("Successfully flushed traces"), Err(e) => { error!("Error sending trace: {e:?}") From c0c59ef27bd5552323fec0c108fec34f55b998f3 Mon Sep 17 00:00:00 2001 From: Julio Gonzalez Date: Mon, 13 May 2024 14:50:41 +0200 Subject: [PATCH 21/64] Refactor trace_utils.rs into smaller units. --- src/trace_processor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index e9a4866..1118512 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -213,7 +213,7 @@ mod tests { assert_eq!( expected_tracer_payload, - tracer_payload.unwrap().get_payloads()[0] + tracer_payload.unwrap().tracer_payloads[0] ); } @@ -283,7 +283,7 @@ mod tests { }; assert_eq!( expected_tracer_payload, - tracer_payload.unwrap().get_payloads()[0] + tracer_payload.unwrap().tracer_payloads[0] ); } } From df0427ac6bd77e9c0b7c9d112ec318528acbb459 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Wed, 29 May 2024 17:31:03 +0000 Subject: [PATCH 22/64] adds azure app service tags to serverless mini agent traces for azure functions --- src/trace_processor.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 1118512..f2eb475 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -12,6 +12,8 @@ use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; +use ddcommon::azure_app_services; + use crate::{ config::Config, http_utils::{self, log_and_create_http_response}, @@ -78,6 +80,35 @@ impl TraceProcessor for ServerlessTraceProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); + match azure_app_services::get_function_metadata() { + Some(aas_metadata) => { + let aas_tags = [ + ("aas.resource.id", aas_metadata.get_resource_id()), + ( + "aas.environment.extension_version", + aas_metadata.get_extension_version(), + ), + ( + "aas.environment.instance_id", + aas_metadata.get_instance_id(), + ), + ( + "aas.environment.instance_name", + aas_metadata.get_instance_name(), + ), + ("aas.environment.os", aas_metadata.get_operating_system()), + ("aas.resource.group", aas_metadata.get_resource_group()), + ("aas.site.name", aas_metadata.get_site_name()), + ("aas.site.kind", aas_metadata.get_site_kind()), + ("aas.site.type", aas_metadata.get_site_type()), + ("aas.subscription.id", aas_metadata.get_subscription_id()), + ]; + aas_tags.into_iter().for_each(|(name, value)| { + span.meta.insert(name.to_string(), value.to_string()); + }); + } + None => (), + } obfuscate_span(span, &config.obfuscation_config); } }, From b861ae9ea62278a1eed74c6ff506d654541e9907 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Thu, 30 May 2024 12:58:54 -0400 Subject: [PATCH 23/64] refactor logic for setting azure span tags in serverless mini agent --- src/trace_processor.rs | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index f2eb475..61cc470 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -9,16 +9,16 @@ use log::info; use tokio::sync::mpsc::Sender; use datadog_trace_obfuscation::obfuscate::obfuscate_span; -use datadog_trace_utils::trace_utils; +use datadog_trace_utils::trace_utils::{self}; use datadog_trace_utils::trace_utils::SendData; -use ddcommon::azure_app_services; - use crate::{ config::Config, http_utils::{self, log_and_create_http_response}, }; +const MINI_AGENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + #[async_trait] pub trait TraceProcessor { /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc @@ -80,35 +80,7 @@ impl TraceProcessor for ServerlessTraceProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - match azure_app_services::get_function_metadata() { - Some(aas_metadata) => { - let aas_tags = [ - ("aas.resource.id", aas_metadata.get_resource_id()), - ( - "aas.environment.extension_version", - aas_metadata.get_extension_version(), - ), - ( - "aas.environment.instance_id", - aas_metadata.get_instance_id(), - ), - ( - "aas.environment.instance_name", - aas_metadata.get_instance_name(), - ), - ("aas.environment.os", aas_metadata.get_operating_system()), - ("aas.resource.group", aas_metadata.get_resource_group()), - ("aas.site.name", aas_metadata.get_site_name()), - ("aas.site.kind", aas_metadata.get_site_kind()), - ("aas.site.type", aas_metadata.get_site_type()), - ("aas.subscription.id", aas_metadata.get_subscription_id()), - ]; - aas_tags.into_iter().for_each(|(name, value)| { - span.meta.insert(name.to_string(), value.to_string()); - }); - } - None => (), - } + trace_utils::enrich_span_with_azure_metadata(span, MINI_AGENT_VERSION); obfuscate_span(span, &config.obfuscation_config); } }, From 97b4ff8514633d1974252edb27c3d8ddc464df81 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Thu, 30 May 2024 13:45:24 -0400 Subject: [PATCH 24/64] apply formatting --- src/trace_processor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 61cc470..4d3b3c7 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -9,8 +9,8 @@ use log::info; use tokio::sync::mpsc::Sender; use datadog_trace_obfuscation::obfuscate::obfuscate_span; -use datadog_trace_utils::trace_utils::{self}; use datadog_trace_utils::trace_utils::SendData; +use datadog_trace_utils::trace_utils::{self}; use crate::{ config::Config, From 026650573a4cfbad81243df8f7a8d3555f211e4a Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Thu, 30 May 2024 21:46:35 -0400 Subject: [PATCH 25/64] add mini agent version to config --- src/config.rs | 20 ++++++++++++-------- src/trace_processor.rs | 4 +--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/config.rs b/src/config.rs index f88931a..4e0816e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,20 +15,21 @@ use datadog_trace_utils::trace_utils; #[derive(Debug)] pub struct Config { - pub function_name: Option, + pub dd_site: String, pub env_type: trace_utils::EnvironmentType, - pub os: String, + pub function_name: Option, pub max_request_content_length: usize, - /// how often to flush traces, in seconds - pub trace_flush_interval: u64, + pub mini_agent_version: String, + pub obfuscation_config: obfuscation_config::ObfuscationConfig, + pub os: String, /// how often to flush stats, in seconds pub stats_flush_interval: u64, - /// timeout for environment verification, in milliseconds - pub verify_env_timeout: u64, + /// how often to flush traces, in seconds + pub trace_flush_interval: u64, pub trace_intake: Endpoint, pub trace_stats_intake: Endpoint, - pub dd_site: String, - pub obfuscation_config: obfuscation_config::ObfuscationConfig, + /// timeout for environment verification, in milliseconds + pub verify_env_timeout: u64, } impl Config { @@ -61,6 +62,8 @@ impl Config { ) })?; + let mini_agent_version: String = env!("CARGO_PKG_VERSION").to_string(); + Ok(Config { function_name: Some(function_name), env_type, @@ -79,6 +82,7 @@ impl Config { api_key: Some(api_key), }, obfuscation_config, + mini_agent_version, }) } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 4d3b3c7..eaef3ca 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -17,8 +17,6 @@ use crate::{ http_utils::{self, log_and_create_http_response}, }; -const MINI_AGENT_VERSION: &str = env!("CARGO_PKG_VERSION"); - #[async_trait] pub trait TraceProcessor { /// Deserializes traces from a hyper request body and sends them through the provided tokio mpsc @@ -80,7 +78,7 @@ impl TraceProcessor for ServerlessTraceProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - trace_utils::enrich_span_with_azure_metadata(span, MINI_AGENT_VERSION); + trace_utils::enrich_span_with_azure_metadata(span, config.mini_agent_version.as_str()); obfuscate_span(span, &config.obfuscation_config); } }, From 0a35e9cca5d817dd87f041676edb10a66fc88e7d Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Thu, 30 May 2024 21:50:06 -0400 Subject: [PATCH 26/64] fix unit test --- src/trace_processor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index eaef3ca..eba14c3 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -152,6 +152,7 @@ mod tests { env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), obfuscation_config: ObfuscationConfig::new().unwrap(), + mini_agent_version: "0.1.0".to_string(), } } From f421ac1cc967d2809a3249064377c43015511a3d Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 31 May 2024 07:27:58 -0400 Subject: [PATCH 27/64] apply formatting --- src/trace_processor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index eba14c3..755b539 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -78,7 +78,10 @@ impl TraceProcessor for ServerlessTraceProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - trace_utils::enrich_span_with_azure_metadata(span, config.mini_agent_version.as_str()); + trace_utils::enrich_span_with_azure_metadata( + span, + config.mini_agent_version.as_str(), + ); obfuscate_span(span, &config.obfuscation_config); } }, From 43820febbfcf66ffd9d935d278fafe0e23954545 Mon Sep 17 00:00:00 2001 From: Julio Gonzalez Date: Fri, 31 May 2024 15:50:53 +0200 Subject: [PATCH 28/64] Add 'src_library' tag to metrics collect in the sidecar. --- src/trace_processor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 1118512..e9a4866 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -213,7 +213,7 @@ mod tests { assert_eq!( expected_tracer_payload, - tracer_payload.unwrap().tracer_payloads[0] + tracer_payload.unwrap().get_payloads()[0] ); } @@ -283,7 +283,7 @@ mod tests { }; assert_eq!( expected_tracer_payload, - tracer_payload.unwrap().tracer_payloads[0] + tracer_payload.unwrap().get_payloads()[0] ); } } From c2892d66fe51ce8d4ab8cb7eb9bd5c5dbc4db0f7 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Mon, 3 Jun 2024 14:22:41 -0400 Subject: [PATCH 29/64] bump serverless mini agent version to 0.4.0 --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5596879..3adf8b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datadog-trace-mini-agent" -description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (GCF and Azure Consumptions plan)" -version = "0.1.0" +description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" +version = "0.4.0" edition = "2021" [dependencies] From 67d6e2cea5d20c7bfd474da99bf8bf779b8d6412 Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Tue, 4 Jun 2024 09:13:06 -0400 Subject: [PATCH 30/64] move trace_util test helpers behind a feature flag (#461) Put trace_test_utils behind a test-utils feature flag To reduce the possibility of test helper functions being used in non-test code. Also, move create_send_data and poll_for_mock_hit functions into trace_test_utils. And update third party license file for test-utils feature in trace-utils crate --- Cargo.toml | 1 + src/trace_processor.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3adf8b1..d0571b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ rmp-serde = "1.1.1" serial_test = "2.0.0" duplicate = "0.4.1" tempfile = "3.3.0" +datadog-trace-utils = { path = "../trace-utils", features=["test-utils"] } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 177973a..5b2fd51 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -124,7 +124,7 @@ mod tests { }; use datadog_trace_protobuf::pb; use datadog_trace_utils::{ - trace_test_utils::{create_test_json_span, create_test_span}, + test_utils::{create_test_json_span, create_test_span}, trace_utils, }; use ddcommon::Endpoint; From 12472513c217188a2f5109894d313d97707b9127 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Fri, 7 Jun 2024 15:12:40 +0200 Subject: [PATCH 31/64] Skip normalization & obfuscation and coalesce instead (#475) * Skip normalization & obfuscation and coalesce instead Turns out the agent ignores the root_span_tags for v0.7 anyway. This is useful to coalesce chunks with common data, allowing us to actually save us from doing redundant requests. As it was currently, the agentful trace sender always sent one trace per request, instead of merging them. We also skip normalization and rely on the agent to do it, for sake of consistency with other tracers and reduce the potentially duplicated work across tracer and agent. We may change that back in future, but for now we've determined it to be the easiest way to work, also with respect to normalization-unaware testing. Signed-off-by: Bob Weinand * Update trace-utils/src/trace_utils.rs Co-authored-by: Pierre Bonet * Update trace-utils/src/trace_utils.rs Co-authored-by: Pierre Bonet * Update trace-mini-agent/src/trace_processor.rs Co-authored-by: Pierre Bonet * Add comment back Signed-off-by: Bob Weinand --------- Signed-off-by: Bob Weinand Co-authored-by: Pierre Bonet --- src/trace_processor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 5b2fd51..3edddca 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -85,6 +85,7 @@ impl TraceProcessor for ServerlessTraceProcessor { obfuscate_span(span, &config.obfuscation_config); } }, + true, // In mini agent, we always send agentless ); let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); From 41789b64a74018311e36b0f3687f17a73216337b Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:51:30 -0400 Subject: [PATCH 32/64] bump version of datadog-trace-mini-agent to 0.4.2 (#502) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d0571b5..809934b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datadog-trace-mini-agent" description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" -version = "0.4.0" +version = "0.4.2" edition = "2021" [dependencies] From 59ad03a3244153a91d00f2ec70a334ef1d5f1a6b Mon Sep 17 00:00:00 2001 From: Julio Gonzalez <107922352+hoolioh@users.noreply.github.com> Date: Tue, 25 Jun 2024 17:04:08 +0200 Subject: [PATCH 33/64] Add support for sending v0.4 traces. (#491) * Add enum to hold v04 and v07 traces. * Modify collect_trace_chunks to handle both types. * Refactor SendData to be able to send both formats. * Change trace exporter, mini-agent and sidecar to handle TracerPayloadCollection type. --- src/trace_processor.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 3edddca..9683a40 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -11,6 +11,7 @@ use tokio::sync::mpsc::Sender; use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_utils::trace_utils::SendData; use datadog_trace_utils::trace_utils::{self}; +use datadog_trace_utils::tracer_payload::TraceEncoding; use crate::{ config::Config, @@ -86,6 +87,7 @@ impl TraceProcessor for ServerlessTraceProcessor { } }, true, // In mini agent, we always send agentless + TraceEncoding::V07, ); let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); @@ -127,6 +129,7 @@ mod tests { use datadog_trace_utils::{ test_utils::{create_test_json_span, create_test_span}, trace_utils, + tracer_payload::TracerPayloadCollection, }; use ddcommon::Endpoint; @@ -217,10 +220,14 @@ mod tests { app_version: "".to_string(), }; - assert_eq!( - expected_tracer_payload, - tracer_payload.unwrap().get_payloads()[0] - ); + let received_payload = + if let TracerPayloadCollection::V07(payload) = tracer_payload.unwrap().get_payloads() { + Some(payload[0].clone()) + } else { + None + }; + + assert_eq!(expected_tracer_payload, received_payload.unwrap()); } #[tokio::test] @@ -287,9 +294,14 @@ mod tests { hostname: "".to_string(), app_version: "".to_string(), }; - assert_eq!( - expected_tracer_payload, - tracer_payload.unwrap().get_payloads()[0] - ); + + let received_payload = + if let TracerPayloadCollection::V07(payload) = tracer_payload.unwrap().get_payloads() { + Some(payload[0].clone()) + } else { + None + }; + + assert_eq!(expected_tracer_payload, received_payload.unwrap()); } } From 01978f513b24c37f4522a778ec26ccb5871b4d19 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 27 Jun 2024 11:05:19 -0400 Subject: [PATCH 34/64] Support v0.5 trace endpoint (#505) * feat: add Lambda * feat: env verifier * feat: set origin tag correctly * debug: what is v5 sending us * revert * feat: v0.5 trace decompression support * fix: remove comment and unwraps * feat: remove erroneous GCP comment from my copypasta * fix: complex type, license for rmpv * feat: Rename methods to use v05. Add v05 string method. Temp debugging log to figure out json string encoding stuff * wip: more debugging * fix: use into_str instead of to_string to avoid string escaping * feat: clean up match * feat: Rename last method to get_v05 nomenclature --- src/env_verifier.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 714456d..8363d22 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -139,6 +139,9 @@ impl EnvVerifier for ServerlessEnvVerifier { .verify_gcp_environment_or_exit(verify_env_timeout) .await; } + trace_utils::EnvironmentType::LambdaFunction => { + trace_utils::MiniAgentMetadata::default() + } } } } From f19bac7e9c92cb52161e90e348ecc4a39ea1e6ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 9 Jul 2024 16:36:46 +0200 Subject: [PATCH 35/64] Enable all benchmarks --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 809934b..dda3de5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,10 @@ name = "datadog-trace-mini-agent" description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" version = "0.4.2" edition = "2021" +autobenches = false + +[lib] +bench = false [dependencies] anyhow = "1.0" From babc5ae0a9c3d43637cb05c74a4353680ad3e62b Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:25:04 -0400 Subject: [PATCH 36/64] bump versions of datadog-serverless-trace-mini-agent and datadog-trace-mini-agent to 0.5.0 (#528) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dda3de5..dc94b62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datadog-trace-mini-agent" description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" -version = "0.4.2" +version = "0.5.0" edition = "2021" autobenches = false From 9b7604c148207a364927c7d08632c9d3c0a69194 Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Wed, 17 Jul 2024 17:18:37 -0400 Subject: [PATCH 37/64] ekump/APMSP-1279 benchmark trace exporting (#531) * Extract logic to build TracerPayloadCollection from sidecar into TracerPayload. This functionality to turn msgpack bytes and header tags into a TracerPayloadCollection should live somewhere more common than the sidecar as it will need to also be used by data-pipeline (and presumably agentless). It only supports v04 at the moment, but should eventually support v05 and v07 in follow up PRs. * Adding some initial benchmarks for deserializing traces from msgpack Deserializing msgpack bytes into an internal representation is currently inefficient. Before work starts on fixing the performance issues we want to set some baseline benchmarks. * Switch the chunk processing to use a trait impl instead of dyn The TraceChunk processor that is used primarily by the mini-agent (but is also created and passed by Sidecar and data-pipeline) was previously a closure that was dynamically dispatched. This was changed to be a trait in order to remove the dynamic-ness and should improve performance and in many cases the compiler should optimize the calls to it away as it is a noop for the Sidecar and data pipeline. Co-authored-by: Bob Weinand --------- Co-authored-by: Bob Weinand --- src/trace_processor.rs | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 9683a40..4f67697 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -9,8 +9,10 @@ use log::info; use tokio::sync::mpsc::Sender; use datadog_trace_obfuscation::obfuscate::obfuscate_span; +use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils::SendData; use datadog_trace_utils::trace_utils::{self}; +use datadog_trace_utils::tracer_payload::TraceChunkProcessor; use datadog_trace_utils::tracer_payload::TraceEncoding; use crate::{ @@ -31,6 +33,28 @@ pub trait TraceProcessor { ) -> http::Result>; } +struct ChunkProcessor { + config: Arc, + mini_agent_metadata: Arc, +} + +impl TraceChunkProcessor for ChunkProcessor { + fn process(&mut self, chunk: &mut pb::TraceChunk, root_span_index: usize) { + trace_utils::set_serverless_root_span_tags( + &mut chunk.spans[root_span_index], + self.config.function_name.clone(), + &self.config.env_type, + ); + for span in chunk.spans.iter_mut() { + trace_utils::enrich_span_with_mini_agent_metadata(span, &self.mini_agent_metadata); + trace_utils::enrich_span_with_azure_metadata( + span, + self.config.mini_agent_version.as_str(), + ); + obfuscate_span(span, &self.config.obfuscation_config); + } + } +} #[derive(Clone)] pub struct ServerlessTraceProcessor {} @@ -71,20 +95,9 @@ impl TraceProcessor for ServerlessTraceProcessor { let payload = trace_utils::collect_trace_chunks( traces, &tracer_header_tags, - |chunk, root_span_index| { - trace_utils::set_serverless_root_span_tags( - &mut chunk.spans[root_span_index], - config.function_name.clone(), - &config.env_type, - ); - for span in chunk.spans.iter_mut() { - trace_utils::enrich_span_with_mini_agent_metadata(span, &mini_agent_metadata); - trace_utils::enrich_span_with_azure_metadata( - span, - config.mini_agent_version.as_str(), - ); - obfuscate_span(span, &config.obfuscation_config); - } + &mut ChunkProcessor { + config: config.clone(), + mini_agent_metadata: mini_agent_metadata.clone(), }, true, // In mini agent, we always send agentless TraceEncoding::V07, From ce790da4e1c35aa3fb13e23e028ae7ee5f465f9b Mon Sep 17 00:00:00 2001 From: Julio Gonzalez <107922352+hoolioh@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:33:23 +0200 Subject: [PATCH 38/64] Implement timeouts for both sending traces and telemetry. (#518) * Add timeout field to Endpoint struct. * Fix Endpoint use to handle the new timeout field. * Implement timeouts in trace-utils and ddtelemetry. * Add FFI call to set the timeout in the endpoint struct. * Make Endpoint instantiation more idiomatic. * Update profling and crashtracker API to use the new endpoint. * Add error handling to set_timeout. --- src/config.rs | 2 ++ src/trace_processor.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/config.rs b/src/config.rs index 4e0816e..647bea8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -76,10 +76,12 @@ impl Config { trace_intake: Endpoint { url: hyper::Uri::from_str(&trace_intake_url).unwrap(), api_key: Some(api_key.clone()), + ..Default::default() }, trace_stats_intake: Endpoint { url: hyper::Uri::from_str(&trace_stats_intake_url).unwrap(), api_key: Some(api_key), + ..Default::default() }, obfuscation_config, mini_agent_version, diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 4f67697..5ec0364 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -163,10 +163,12 @@ mod tests { trace_intake: Endpoint { url: hyper::Uri::from_static("https://trace.agent.notdog.com/traces"), api_key: Some("dummy_api_key".into()), + ..Default::default() }, trace_stats_intake: Endpoint { url: hyper::Uri::from_static("https://trace.agent.notdog.com/stats"), api_key: Some("dummy_api_key".into()), + ..Default::default() }, dd_site: "datadoghq.com".to_string(), env_type: trace_utils::EnvironmentType::CloudFunction, From c2d9602ef6acb77fb3ce4774264314c214319c93 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Thu, 25 Jul 2024 16:38:40 -0400 Subject: [PATCH 39/64] Fixes to make clippy version 1.80 happy (#550) * cargo clippy fix for 1.80 * make windows happy * windows unexpected cfgs * typo --- src/http_utils.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/http_utils.rs b/src/http_utils.rs index 701af8d..d2afb16 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -11,8 +11,7 @@ use serde_json::json; /// Does two things: /// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be -/// written, -/// otherwise error will be written. +/// written, otherwise error will be written. /// 2. Returns the given message in the body of JSON response with the given status code. /// /// Response body format: From 932a157861da18becdee627b6e5ea9544a054004 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 31 Jul 2024 07:43:25 -0400 Subject: [PATCH 40/64] [Serverless Mini Agent] Add _dd.mini_agent_version tag to all spans for Azure Functions, Google Cloud Functions, and Azure Spring Apps (#548) * add _dd.mini_agent_version to spans in all environments * apply formatting --- Cargo.toml | 6 ++++-- src/config.rs | 4 ---- src/env_verifier.rs | 2 ++ src/trace_processor.rs | 6 +----- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dc94b62..483e230 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "datadog-trace-mini-agent" description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" -version = "0.5.0" -edition = "2021" +edition.workspace = true +version.workspace = true +rust-version.workspace = true +license.workspace = true autobenches = false [lib] diff --git a/src/config.rs b/src/config.rs index 647bea8..418c649 100644 --- a/src/config.rs +++ b/src/config.rs @@ -19,7 +19,6 @@ pub struct Config { pub env_type: trace_utils::EnvironmentType, pub function_name: Option, pub max_request_content_length: usize, - pub mini_agent_version: String, pub obfuscation_config: obfuscation_config::ObfuscationConfig, pub os: String, /// how often to flush stats, in seconds @@ -62,8 +61,6 @@ impl Config { ) })?; - let mini_agent_version: String = env!("CARGO_PKG_VERSION").to_string(); - Ok(Config { function_name: Some(function_name), env_type, @@ -84,7 +81,6 @@ impl Config { ..Default::default() }, obfuscation_config, - mini_agent_version, }) } } diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 8363d22..4cd2fc1 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -117,6 +117,7 @@ impl ServerlessEnvVerifier { gcp_region: Some(get_region_from_gcp_region_string( gcp_metadata.instance.region, )), + version: trace_utils::MiniAgentMetadata::default().version, } } } @@ -466,6 +467,7 @@ mod tests { trace_utils::MiniAgentMetadata { gcp_project_id: Some("unknown".to_string()), gcp_region: Some("unknown".to_string()), + version: None } ); } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 5ec0364..feae500 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -47,10 +47,7 @@ impl TraceChunkProcessor for ChunkProcessor { ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &self.mini_agent_metadata); - trace_utils::enrich_span_with_azure_metadata( - span, - self.config.mini_agent_version.as_str(), - ); + trace_utils::enrich_span_with_azure_metadata(span); obfuscate_span(span, &self.config.obfuscation_config); } } @@ -174,7 +171,6 @@ mod tests { env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), obfuscation_config: ObfuscationConfig::new().unwrap(), - mini_agent_version: "0.1.0".to_string(), } } From d82e016d391b2802f97fbe3e100e3025f6fe3a28 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 31 Jul 2024 08:24:08 -0400 Subject: [PATCH 41/64] [Serverless Mini Agent] Run in Azure Spring Apps (#547) * add azure spring app environment type for serverless mini agent * add config.statsd_port to mini agent info endpoint * update mini agent trace endpoint status code and response body to work with java tracer * use different environment variable to identify azure spring apps * only update http response for success responses to traces endpoint * address lint errors * updates comment and formatting * update trace-mini-agent description to include Azure Spring Apps * fix formatting --- Cargo.toml | 2 +- src/env_verifier.rs | 3 +++ src/http_utils.rs | 20 ++++++++++++++++++++ src/mini_agent.rs | 3 +++ src/trace_processor.rs | 6 +++--- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 483e230..1ffc2bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-trace-mini-agent" -description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions and Azure Functions)" +description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions, Azure Functions, and Azure Spring Apps)" edition.workspace = true version.workspace = true rust-version.workspace = true diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 4cd2fc1..3400006 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -140,6 +140,9 @@ impl EnvVerifier for ServerlessEnvVerifier { .verify_gcp_environment_or_exit(verify_env_timeout) .await; } + trace_utils::EnvironmentType::AzureSpringApp => { + trace_utils::MiniAgentMetadata::default() + } trace_utils::EnvironmentType::LambdaFunction => { trace_utils::MiniAgentMetadata::default() } diff --git a/src/http_utils.rs b/src/http_utils.rs index d2afb16..523736d 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -31,6 +31,26 @@ pub fn log_and_create_http_response( Response::builder().status(status).body(Body::from(body)) } +/// Does two things: +/// 1. Logs the given message +/// 2. Returns the rate_by_service map to use to set the sampling priority in the body of JSON +/// response with the given status code. +/// +/// Response body format: +/// { +/// "rate_by_service": { +/// "service:,env:":1 +/// } +/// } +pub fn log_and_create_traces_success_http_response( + message: &str, + status: StatusCode, +) -> http::Result> { + info!("{message}"); + let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); + Response::builder().status(status).body(Body::from(body)) +} + /// Takes a request's header map, and verifies that the "content-length" header is present, valid, /// and less than the given max_content_length. /// diff --git a/src/mini_agent.rs b/src/mini_agent.rs index e42103b..e62b53c 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -192,6 +192,9 @@ impl MiniAgent { INFO_ENDPOINT_PATH ], "client_drop_p0s": true, + "config": { + "statsd_port": MINI_AGENT_PORT + } } ); Response::builder() diff --git a/src/trace_processor.rs b/src/trace_processor.rs index feae500..be1fafb 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -17,7 +17,7 @@ use datadog_trace_utils::tracer_payload::TraceEncoding; use crate::{ config::Config, - http_utils::{self, log_and_create_http_response}, + http_utils::{self, log_and_create_http_response, log_and_create_traces_success_http_response}, }; #[async_trait] @@ -105,9 +105,9 @@ impl TraceProcessor for ServerlessTraceProcessor { // send trace payload to our trace flusher match tx.send(send_data).await { Ok(_) => { - return log_and_create_http_response( + return log_and_create_traces_success_http_response( "Successfully buffered traces to be flushed.", - StatusCode::ACCEPTED, + StatusCode::OK, ); } Err(err) => { From 208edec61b47f7f75f36b9b71ee4ded2ef679752 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Mon, 5 Aug 2024 10:10:53 -0400 Subject: [PATCH 42/64] [SVLS-5049] It is okay to have a stats payload without stats (#567) --- src/stats_processor.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 5d9ee5d..2d2dfad 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -62,12 +62,14 @@ impl StatsProcessor for ServerlessStatsProcessor { } }; - let start = SystemTime::now(); - let timestamp = start - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - stats.stats[0].start = timestamp as u64; + if !stats.stats.is_empty() { + let start = SystemTime::now(); + let timestamp = start + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + stats.stats[0].start = timestamp as u64; + } // send trace payload to our trace flusher match tx.send(stats).await { From 78aabe54103de831b699281f50059aaa7cbcb3ae Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Fri, 13 Sep 2024 08:54:54 -0400 Subject: [PATCH 43/64] [Serverless Mini Agent] Use DogStatsD in Serverless (#616) * use dogstatsd in serverless * convert env var value to lowercase and nest dogstatsd logic * refactor to move loop outside of conditional dogstatsd block * add environment variables for hardcoded values and refactor start_dogstatsd * fix unit test * add env var for default dogstatsd port --- src/config.rs | 37 +++++++++++++++++++++++++++++++++++++ src/mini_agent.rs | 7 +++---- src/trace_processor.rs | 1 + 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 418c649..101e2af 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,9 +13,12 @@ use datadog_trace_utils::config_utils::{ }; use datadog_trace_utils::trace_utils; +const DEFAULT_DOGSTATSD_PORT: u16 = 8125; + #[derive(Debug)] pub struct Config { pub dd_site: String, + pub dd_dogstatsd_port: u16, pub env_type: trace_utils::EnvironmentType, pub function_name: Option, pub max_request_content_length: usize, @@ -41,6 +44,10 @@ impl Config { anyhow::anyhow!("Unable to identify environment. Shutting down Mini Agent.") })?; + let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") + .ok() + .and_then(|port| port.parse::().ok()) + .unwrap_or(DEFAULT_DOGSTATSD_PORT); let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); // construct the trace & trace stats intake urls based on DD_SITE env var (to flush traces & @@ -69,6 +76,7 @@ impl Config { trace_flush_interval: 3, stats_flush_interval: 3, verify_env_timeout: 100, + dd_dogstatsd_port, dd_site, trace_intake: Endpoint { url: hyper::Uri::from_str(&trace_intake_url).unwrap(), @@ -207,4 +215,33 @@ mod tests { env::remove_var("DD_APM_DD_URL"); env::remove_var("K_SERVICE"); } + + #[test] + #[serial] + fn test_default_dogstatsd_port() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_dogstatsd_port, 8125); + env::remove_var("DD_API_KEY"); + env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); + } + + #[test] + #[serial] + fn test_custom_dogstatsd_port() { + env::set_var("DD_API_KEY", "_not_a_real_key_"); + env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); + env::set_var("DD_DOGSTATSD_PORT", "18125"); + let config_res = config::Config::new(); + println!("{:?}", config_res); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_dogstatsd_port, 18125); + env::remove_var("DD_API_KEY"); + env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); + env::remove_var("DD_DOGSTATSD_PORT"); + } } diff --git a/src/mini_agent.rs b/src/mini_agent.rs index e62b53c..e6e24b5 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -34,7 +34,6 @@ pub struct MiniAgent { } impl MiniAgent { - #[tokio::main] pub async fn start_mini_agent(&self) -> Result<(), Box> { let now = Instant::now(); @@ -168,7 +167,7 @@ impl MiniAgent { ), } } - (_, INFO_ENDPOINT_PATH) => match Self::info_handler() { + (_, INFO_ENDPOINT_PATH) => match Self::info_handler(config.dd_dogstatsd_port) { Ok(res) => Ok(res), Err(err) => log_and_create_http_response( &format!("Info endpoint error: {err}"), @@ -183,7 +182,7 @@ impl MiniAgent { } } - fn info_handler() -> http::Result> { + fn info_handler(dd_dogstatsd_port: u16) -> http::Result> { let response_json = json!( { "endpoints": [ @@ -193,7 +192,7 @@ impl MiniAgent { ], "client_drop_p0s": true, "config": { - "statsd_port": MINI_AGENT_PORT + "statsd_port": dd_dogstatsd_port } } ); diff --git a/src/trace_processor.rs b/src/trace_processor.rs index be1fafb..59baca6 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -168,6 +168,7 @@ mod tests { ..Default::default() }, dd_site: "datadoghq.com".to_string(), + dd_dogstatsd_port: 8125, env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), obfuscation_config: ObfuscationConfig::new().unwrap(), From 0bd2b173c1746282c1a819b6502bcdb46e449c9c Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Thu, 19 Sep 2024 12:29:56 -0400 Subject: [PATCH 44/64] reduced allocation v04 span representation (#598) Leverage tinybytes to perform zero-copy msgpack decoding of v04 spans within trace-utils for use by data-pipeline and the sidecar. This was driven by the need to reduce memory footprint of the sidecar in PHP. --------- Signed-off-by: Bob Weinand Co-authored-by: Bob Weinand --- src/trace_processor.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 59baca6..dbf7c8f 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -12,8 +12,7 @@ use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils::SendData; use datadog_trace_utils::trace_utils::{self}; -use datadog_trace_utils::tracer_payload::TraceChunkProcessor; -use datadog_trace_utils::tracer_payload::TraceEncoding; +use datadog_trace_utils::tracer_payload::{TraceChunkProcessor, TraceCollection}; use crate::{ config::Config, @@ -90,14 +89,13 @@ impl TraceProcessor for ServerlessTraceProcessor { }; let payload = trace_utils::collect_trace_chunks( - traces, + TraceCollection::V07(traces), &tracer_header_tags, &mut ChunkProcessor { config: config.clone(), mini_agent_metadata: mini_agent_metadata.clone(), }, true, // In mini agent, we always send agentless - TraceEncoding::V07, ); let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); From 69d25e744677fd17e33d7e55cd1294c9f89f7a51 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 2 Oct 2024 15:53:42 -0400 Subject: [PATCH 45/64] Support http-proxy for trace agent, remove proxy from dsd (#658) * feat: support HTTPS_PROXY for traces * fix: https_proxy only. * fix: maybe native-tls works in lambda? * feat: try rust-tls * fix: rusttls oops * fix: reee it's rustls why * fix: default-features must be false * WIP: debug log, will revert * fix: reqwest honors system proxies, hyper doesn't seem to. Only proxy https traffic * fix: revert hyper-proxy, just use system proxy * fix: revert proxy, use system * fix: revert hyper-proxy, use system proxy * fix: revert senddata proxy change from tests * Revert "fix: revert senddata proxy change from tests" This reverts commit 105000853f86dd46c39914434907452d9ca05b60. * Revert "fix: revert hyper-proxy, use system proxy" This reverts commit d9ebdc7e3cd68f046a4e8d981ac0564b34458983. * Revert "fix: revert proxy, use system" This reverts commit e4a8e18c14b56f8b889aec23b40cf3ccaf511639. * Revert "fix: revert hyper-proxy, just use system proxy" This reverts commit f8ed3005f75bdb3a4a4fb35dc52f499206b8d2b9. * fix: re-commit tests * fix: fmt * feat: license * feat: Wrap proxy in feature flag * fix: fmt * fix: not sure why the arg is needed in create_send_data * fix: no api changes for public interfaces * fix: None * fix: allow unused * fix: None for update_send_results_example * fix: remove unused error import --- Cargo.toml | 2 +- src/config.rs | 2 ++ src/trace_processor.rs | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1ffc2bb..56bf801 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" ddcommon = { path = "../ddcommon" } datadog-trace-protobuf = { path = "../trace-protobuf" } -datadog-trace-utils = { path = "../trace-utils" } +datadog-trace-utils = { path = "../trace-utils", features = ["proxy"] } datadog-trace-normalization = { path = "../trace-normalization" } datadog-trace-obfuscation = { path = "../trace-obfuscation" } diff --git a/src/config.rs b/src/config.rs index 101e2af..a2e7fe1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -32,6 +32,7 @@ pub struct Config { pub trace_stats_intake: Endpoint, /// timeout for environment verification, in milliseconds pub verify_env_timeout: u64, + pub proxy_url: Option, } impl Config { @@ -89,6 +90,7 @@ impl Config { ..Default::default() }, obfuscation_config, + proxy_url: env::var("HTTPS_PROXY").ok(), }) } } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index dbf7c8f..778e4da 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -98,7 +98,13 @@ impl TraceProcessor for ServerlessTraceProcessor { true, // In mini agent, we always send agentless ); - let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); + let send_data = SendData::new( + body_size, + payload, + tracer_header_tags, + &config.trace_intake, + config.proxy_url.clone(), + ); // send trace payload to our trace flusher match tx.send(send_data).await { @@ -170,6 +176,7 @@ mod tests { env_type: trace_utils::EnvironmentType::CloudFunction, os: "linux".to_string(), obfuscation_config: ObfuscationConfig::new().unwrap(), + proxy_url: None, } } From 58e1424d21b7a1cdfff9a7cae980050d78975b39 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:37:02 -0400 Subject: [PATCH 46/64] [Serverless Mini Agent] Add Span Tags for Azure Spring Apps (#672) * add span tags for azure spring apps * fix unit test --- src/config.rs | 6 +++--- src/env_verifier.rs | 16 +++++++++++++--- src/trace_processor.rs | 6 +++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/config.rs b/src/config.rs index a2e7fe1..bcd4a66 100644 --- a/src/config.rs +++ b/src/config.rs @@ -20,7 +20,7 @@ pub struct Config { pub dd_site: String, pub dd_dogstatsd_port: u16, pub env_type: trace_utils::EnvironmentType, - pub function_name: Option, + pub app_name: Option, pub max_request_content_length: usize, pub obfuscation_config: obfuscation_config::ObfuscationConfig, pub os: String, @@ -41,7 +41,7 @@ impl Config { .map_err(|_| anyhow::anyhow!("DD_API_KEY environment variable is not set"))? .into(); - let (function_name, env_type) = read_cloud_env().ok_or_else(|| { + let (app_name, env_type) = read_cloud_env().ok_or_else(|| { anyhow::anyhow!("Unable to identify environment. Shutting down Mini Agent.") })?; @@ -70,7 +70,7 @@ impl Config { })?; Ok(Config { - function_name: Some(function_name), + app_name: Some(app_name), env_type, os: env::consts::OS.to_string(), max_request_content_length: 10 * 1024 * 1024, // 10MB in Bytes diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 3400006..393952d 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use hyper::{Body, Client, Method, Request, Response}; use log::{debug, error}; use serde::{Deserialize, Serialize}; +use std::env; use std::fs; use std::path::Path; use std::process; @@ -113,6 +114,9 @@ impl ServerlessEnvVerifier { } }; trace_utils::MiniAgentMetadata { + azure_spring_app_hostname: trace_utils::MiniAgentMetadata::default() + .azure_spring_app_hostname, + azure_spring_app_name: trace_utils::MiniAgentMetadata::default().azure_spring_app_name, gcp_project_id: Some(gcp_metadata.project.project_id), gcp_region: Some(get_region_from_gcp_region_string( gcp_metadata.instance.region, @@ -140,9 +144,13 @@ impl EnvVerifier for ServerlessEnvVerifier { .verify_gcp_environment_or_exit(verify_env_timeout) .await; } - trace_utils::EnvironmentType::AzureSpringApp => { - trace_utils::MiniAgentMetadata::default() - } + trace_utils::EnvironmentType::AzureSpringApp => trace_utils::MiniAgentMetadata { + azure_spring_app_hostname: env::var("HOSTNAME").ok(), + azure_spring_app_name: env::var("ASCSVCRT_SPRING__APPLICATION__NAME").ok(), + gcp_project_id: trace_utils::MiniAgentMetadata::default().gcp_project_id, + gcp_region: trace_utils::MiniAgentMetadata::default().gcp_region, + version: trace_utils::MiniAgentMetadata::default().version, + }, trace_utils::EnvironmentType::LambdaFunction => { trace_utils::MiniAgentMetadata::default() } @@ -468,6 +476,8 @@ mod tests { assert_eq!( res, trace_utils::MiniAgentMetadata { + azure_spring_app_hostname: None, + azure_spring_app_name: None, gcp_project_id: Some("unknown".to_string()), gcp_region: Some("unknown".to_string()), version: None diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 778e4da..389719a 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -41,12 +41,12 @@ impl TraceChunkProcessor for ChunkProcessor { fn process(&mut self, chunk: &mut pb::TraceChunk, root_span_index: usize) { trace_utils::set_serverless_root_span_tags( &mut chunk.spans[root_span_index], - self.config.function_name.clone(), + self.config.app_name.clone(), &self.config.env_type, ); for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &self.mini_agent_metadata); - trace_utils::enrich_span_with_azure_metadata(span); + trace_utils::enrich_span_with_azure_function_metadata(span); obfuscate_span(span, &self.config.obfuscation_config); } } @@ -156,7 +156,7 @@ mod tests { fn create_test_config() -> Config { Config { - function_name: Some("dummy_function_name".to_string()), + app_name: Some("dummy_function_name".to_string()), max_request_content_length: 10 * 1024 * 1024, trace_flush_interval: 3, stats_flush_interval: 3, From dd7f2b8a101b436d3cb4e3a821e9a2a2def41d71 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 16 Oct 2024 16:28:20 -0400 Subject: [PATCH 47/64] feat: Prefer DD_PROXY_HTTPS over HTTPS_PROXY (#673) * feat: Prefer DD_PROXY_HTTPS over HTTPS_PROXY * fix: no proxy on ints * fix: clippy thx --- src/config.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index bcd4a66..d93a745 100644 --- a/src/config.rs +++ b/src/config.rs @@ -90,7 +90,9 @@ impl Config { ..Default::default() }, obfuscation_config, - proxy_url: env::var("HTTPS_PROXY").ok(), + proxy_url: env::var("DD_PROXY_HTTPS") + .or_else(|_| env::var("HTTPS_PROXY")) + .ok(), }) } } From c2c6b1b9cd185bff6c3de5b25ac5093931dfc510 Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Thu, 17 Oct 2024 16:55:23 +0100 Subject: [PATCH 48/64] Enable backports/deprecated features on hyper crate These features will help us migrate to 1.x, as suggested in the migration guide: https://hyper.rs/guides/1/upgrading/ . --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56bf801..0de5d94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ bench = false [dependencies] anyhow = "1.0" -hyper = { version = "0.14", default-features = false, features = ["server"] } +hyper = { version = "0.14", default-features = false, features = ["server", "backports", "deprecated"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" log = "0.4" From 48a976ea0f2770d6034fa622a4cfb14b4ddad6fe Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Fri, 18 Oct 2024 09:26:34 +0100 Subject: [PATCH 49/64] Fix all compatibility warnings emitted by hyper crate --- src/env_verifier.rs | 3 ++- src/http_utils.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/env_verifier.rs b/src/env_verifier.rs index 393952d..b00cb7e 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; +use hyper::body::HttpBody; use hyper::{Body, Client, Method, Request, Response}; use log::{debug, error}; use serde::{Deserialize, Serialize}; @@ -233,7 +234,7 @@ async fn ensure_gcp_function_environment( } async fn get_gcp_metadata_from_body(body: hyper::Body) -> anyhow::Result { - let bytes = hyper::body::to_bytes(body).await?; + let bytes = body.collect().await?.to_bytes(); let body_str = String::from_utf8(bytes.to_vec())?; let gcp_metadata: GCPMetadata = serde_json::from_str(&body_str)?; Ok(gcp_metadata) diff --git a/src/http_utils.rs b/src/http_utils.rs index 523736d..b5cf576 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -99,6 +99,7 @@ pub fn verify_request_content_length( #[cfg(test)] mod tests { + use hyper::body::HttpBody; use hyper::header; use hyper::Body; use hyper::HeaderMap; @@ -115,7 +116,7 @@ mod tests { async fn get_response_body_as_string(response: Response) -> String { let body = response.into_body(); - let bytes = hyper::body::to_bytes(body).await.unwrap(); + let bytes = body.collect().await.unwrap().to_bytes(); String::from_utf8(bytes.into_iter().collect()).unwrap() } From 2fa4a79f16da5dbe76f5edef3ecc9d668785e4fa Mon Sep 17 00:00:00 2001 From: Julio Gonzalez <107922352+hoolioh@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:33:59 +0100 Subject: [PATCH 50/64] Refactor proxy handling. --- src/trace_flusher.rs | 12 ++++++++---- src/trace_processor.rs | 8 +------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index ba28091..7987e76 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -17,7 +17,7 @@ pub trait TraceFlusher { /// implementing flushing logic that calls flush_traces. async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver); /// Flushes traces to the Datadog trace intake. - async fn flush_traces(&self, traces: Vec); + async fn flush_traces(&self, traces: Vec, config: Arc); } #[derive(Clone)] @@ -43,20 +43,24 @@ impl TraceFlusher for ServerlessTraceFlusher { let mut buffer = buffer_consumer.lock().await; if !buffer.is_empty() { - self.flush_traces(buffer.to_vec()).await; + self.flush_traces(buffer.to_vec(), config.clone()).await; buffer.clear(); } } } - async fn flush_traces(&self, traces: Vec) { + async fn flush_traces(&self, traces: Vec, config: Arc) { if traces.is_empty() { return; } info!("Flushing {} traces", traces.len()); for traces in trace_utils::coalesce_send_data(traces) { - match traces.send().await.last_result { + match traces + .send_proxy(config.proxy_url.as_deref()) + .await + .last_result + { Ok(_) => info!("Successfully flushed traces"), Err(e) => { error!("Error sending trace: {e:?}") diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 389719a..2ff8611 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -98,13 +98,7 @@ impl TraceProcessor for ServerlessTraceProcessor { true, // In mini agent, we always send agentless ); - let send_data = SendData::new( - body_size, - payload, - tracer_header_tags, - &config.trace_intake, - config.proxy_url.clone(), - ); + let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); // send trace payload to our trace flusher match tx.send(send_data).await { From ad3a549d1e734345e3cf59e96cf6bcd3e2f9e094 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 20 Nov 2024 15:13:41 -0500 Subject: [PATCH 51/64] [chore] Use elapsed() if possible when calculating durations (#750) --- src/stats_processor.rs | 8 ++------ src/trace_processor.rs | 11 ++--------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 2d2dfad..38539f5 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; @@ -63,11 +63,7 @@ impl StatsProcessor for ServerlessStatsProcessor { }; if !stats.stats.is_empty() { - let start = SystemTime::now(); - let timestamp = start - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); + let timestamp = UNIX_EPOCH.elapsed().unwrap_or_default().as_nanos(); stats.stats[0].start = timestamp as u64; } diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 2ff8611..07360df 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -122,11 +122,7 @@ impl TraceProcessor for ServerlessTraceProcessor { mod tests { use datadog_trace_obfuscation::obfuscation_config::ObfuscationConfig; use hyper::Request; - use std::{ - collections::HashMap, - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, - }; + use std::{collections::HashMap, sync::Arc, time::UNIX_EPOCH}; use tokio::sync::mpsc::{self, Receiver, Sender}; use crate::{ @@ -142,10 +138,7 @@ mod tests { use ddcommon::Endpoint; fn get_current_timestamp_nanos() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as i64 + UNIX_EPOCH.elapsed().unwrap().as_nanos() as i64 } fn create_test_config() -> Config { From 3877fab9a905a072c0d77ca454e8efacd2540bc7 Mon Sep 17 00:00:00 2001 From: Levi Morrison Date: Mon, 2 Dec 2024 01:39:27 -0700 Subject: [PATCH 52/64] fix(alloc): potentially dangling temporary (#772) * fix: potentially dangling temporary I was trying out the Rust 2024 edition which required a new nightly. There's now a warning for dangling_pointers_from_temporaries. This could be a false positive but I am not certain. * test: skip problematic miri tests on MacOS These are all due to kqueue, example: ``` error: unsupported operation: can't call foreign function `kqueue` on OS `macos` --> /levi.morrison/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mio-1.0.2/src/sys/unix/selector/kqueue.rs:76:48 | 76 | let kq = unsafe { OwnedFd::from_raw_fd(syscall!(kqueue())?) }; | ^^^^^^^^^^^^^^^^^^ can't call foreign function `kqueue` on OS `macos` ``` * test: fix env test when running locally If the local env has this set, it fails. --- src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.rs b/src/config.rs index d93a745..4bb943e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -122,6 +122,7 @@ mod tests { #[test] #[serial] fn test_error_if_no_api_key_env_var() { + env::remove_var("DD_API_KEY"); let config = config::Config::new(); assert!(config.is_err()); assert_eq!( From 80679235d2ecd99eb36f50748cf3cf07d1aef935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:15:15 -0600 Subject: [PATCH 53/64] [trace-mini-agent] change logging crate (#799) * `log` => `tracing` * fmt --- Cargo.toml | 2 +- src/env_verifier.rs | 2 +- src/http_utils.rs | 2 +- src/mini_agent.rs | 2 +- src/stats_flusher.rs | 2 +- src/stats_processor.rs | 2 +- src/trace_flusher.rs | 2 +- src/trace_processor.rs | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0de5d94..381691f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ anyhow = "1.0" hyper = { version = "0.14", default-features = false, features = ["server", "backports", "deprecated"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" -log = "0.4" +tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" ddcommon = { path = "../ddcommon" } diff --git a/src/env_verifier.rs b/src/env_verifier.rs index b00cb7e..f06f30d 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -4,7 +4,6 @@ use async_trait::async_trait; use hyper::body::HttpBody; use hyper::{Body, Client, Method, Request, Response}; -use log::{debug, error}; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -12,6 +11,7 @@ use std::path::Path; use std::process; use std::sync::Arc; use std::time::{Duration, Instant}; +use tracing::{debug, error}; use datadog_trace_utils::trace_utils; diff --git a/src/http_utils.rs b/src/http_utils.rs index b5cf576..b17162f 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -6,8 +6,8 @@ use hyper::{ http::{self, HeaderMap}, Body, Response, StatusCode, }; -use log::{error, info}; use serde_json::json; +use tracing::{error, info}; /// Does two things: /// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be diff --git a/src/mini_agent.rs b/src/mini_agent.rs index e6e24b5..9d2b963 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -3,13 +3,13 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; -use log::{debug, error, info}; use serde_json::json; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; +use tracing::{debug, error, info}; use crate::http_utils::log_and_create_http_response; use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index 9d7bed7..e2fa339 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; -use log::{debug, error, info}; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; +use tracing::{debug, error, info}; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 38539f5..ca431df 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -6,8 +6,8 @@ use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; -use log::info; use tokio::sync::mpsc::Sender; +use tracing::info; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index 7987e76..b93e023 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; -use log::{error, info}; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; +use tracing::{error, info}; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 07360df..244f613 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; -use log::info; use tokio::sync::mpsc::Sender; +use tracing::info; use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; From 696e009e0d35ee70ce5dc3863e1875c3fe646d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:57:04 -0600 Subject: [PATCH 54/64] change logs to be `debug` (#806) info is too noisy for customers --- src/http_utils.rs | 6 +++--- src/mini_agent.rs | 4 ++-- src/stats_flusher.rs | 6 +++--- src/stats_processor.rs | 4 ++-- src/trace_flusher.rs | 6 +++--- src/trace_processor.rs | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/http_utils.rs b/src/http_utils.rs index b17162f..01db15c 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -7,7 +7,7 @@ use hyper::{ Body, Response, StatusCode, }; use serde_json::json; -use tracing::{error, info}; +use tracing::{debug, error}; /// Does two things: /// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be @@ -23,7 +23,7 @@ pub fn log_and_create_http_response( status: StatusCode, ) -> http::Result> { if status.is_success() { - info!("{message}"); + debug!("{message}"); } else { error!("{message}"); } @@ -46,7 +46,7 @@ pub fn log_and_create_traces_success_http_response( message: &str, status: StatusCode, ) -> http::Result> { - info!("{message}"); + debug!("{message}"); let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); Response::builder().status(status).body(Body::from(body)) } diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 9d2b963..b6f548f 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -9,7 +9,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tracing::{debug, error, info}; +use tracing::{debug, error}; use crate::http_utils::log_and_create_http_response; use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; @@ -121,7 +121,7 @@ impl MiniAgent { let server = server_builder.serve(make_svc); - info!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); + debug!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); debug!( "Time taken start the Mini Agent: {} ms", now.elapsed().as_millis() diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index e2fa339..ac8d59a 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; -use tracing::{debug, error, info}; +use tracing::{debug, error}; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; @@ -61,7 +61,7 @@ impl StatsFlusher for ServerlessStatsFlusher { if stats.is_empty() { return; } - info!("Flushing {} stats", stats.len()); + debug!("Flushing {} stats", stats.len()); let stats_payload = stats_utils::construct_stats_payload(stats); @@ -82,7 +82,7 @@ impl StatsFlusher for ServerlessStatsFlusher { ) .await { - Ok(_) => info!("Successfully flushed stats"), + Ok(_) => debug!("Successfully flushed stats"), Err(e) => { error!("Error sending stats: {e:?}") } diff --git a/src/stats_processor.rs b/src/stats_processor.rs index ca431df..28674eb 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -7,7 +7,7 @@ use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; use tokio::sync::mpsc::Sender; -use tracing::info; +use tracing::debug; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; @@ -38,7 +38,7 @@ impl StatsProcessor for ServerlessStatsProcessor { req: Request, tx: Sender, ) -> http::Result> { - info!("Recieved trace stats to process"); + debug!("Received trace stats to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index b93e023..7004b33 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; -use tracing::{error, info}; +use tracing::{debug, error}; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; @@ -53,7 +53,7 @@ impl TraceFlusher for ServerlessTraceFlusher { if traces.is_empty() { return; } - info!("Flushing {} traces", traces.len()); + debug!("Flushing {} traces", traces.len()); for traces in trace_utils::coalesce_send_data(traces) { match traces @@ -61,7 +61,7 @@ impl TraceFlusher for ServerlessTraceFlusher { .await .last_result { - Ok(_) => info!("Successfully flushed traces"), + Ok(_) => debug!("Successfully flushed traces"), Err(e) => { error!("Error sending trace: {e:?}") // TODO: Retries diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 244f613..9d75e21 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; use tokio::sync::mpsc::Sender; -use tracing::info; +use tracing::debug; use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; @@ -63,7 +63,7 @@ impl TraceProcessor for ServerlessTraceProcessor { tx: Sender, mini_agent_metadata: Arc, ) -> http::Result> { - info!("Recieved traces to process"); + debug!("Received traces to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( From 5d83ae2b0b77e2c6e5a23d0ec1fee505218d8f83 Mon Sep 17 00:00:00 2001 From: Nina Rei <31524679+nina9753@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:35:28 -0500 Subject: [PATCH 55/64] Add _dd.gcrfx.resource_name to serverless compatibility layer for Google Cloud Functions (#770) * Add _dd.gcrfx.resource_name to mini agent for Google Cloud Functions * fix lint * fix lint * add create_test_gcp_span * add create_test_gcp_span * update test_process_trace * update tests * simplify function * reformat file * reformat file * FIX TESTS!!!!!! * move helpers function * move helpers function * lint * update code from comments * update enrich_span_with_google_cloud_function_metadata * lint * update tag to remove _dd prefix * lint * update enrich_span_with_google_cloud_function_metadata to todo instead of return --- src/trace_processor.rs | 44 ++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 9d75e21..14564e7 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -10,8 +10,8 @@ use tracing::debug; use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; -use datadog_trace_utils::trace_utils::SendData; use datadog_trace_utils::trace_utils::{self}; +use datadog_trace_utils::trace_utils::{EnvironmentType, SendData}; use datadog_trace_utils::tracer_payload::{TraceChunkProcessor, TraceCollection}; use crate::{ @@ -47,6 +47,13 @@ impl TraceChunkProcessor for ChunkProcessor { for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &self.mini_agent_metadata); trace_utils::enrich_span_with_azure_function_metadata(span); + if let EnvironmentType::CloudFunction = &self.config.env_type { + trace_utils::enrich_span_with_google_cloud_function_metadata( + span, + &self.mini_agent_metadata, + self.config.app_name.clone(), + ); + } obfuscate_span(span, &self.config.obfuscation_config); } } @@ -130,10 +137,10 @@ mod tests { trace_processor::{self, TraceProcessor}, }; use datadog_trace_protobuf::pb; + use datadog_trace_utils::test_utils::{create_test_gcp_json_span, create_test_gcp_span}; + use datadog_trace_utils::trace_utils::MiniAgentMetadata; use datadog_trace_utils::{ - test_utils::{create_test_json_span, create_test_span}, - trace_utils, - tracer_payload::TracerPayloadCollection, + test_utils::create_test_json_span, trace_utils, tracer_payload::TracerPayloadCollection, }; use ddcommon::Endpoint; @@ -167,6 +174,16 @@ mod tests { } } + fn create_test_metadata() -> MiniAgentMetadata { + MiniAgentMetadata { + azure_spring_app_hostname: Default::default(), + azure_spring_app_name: Default::default(), + gcp_project_id: Some("dummy_project_id".to_string()), + gcp_region: Some("dummy_region_west".to_string()), + version: Some("dummy_version".to_string()), + } + } + #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_process_trace() { @@ -196,7 +213,7 @@ mod tests { Arc::new(create_test_config()), request, tx, - Arc::new(trace_utils::MiniAgentMetadata::default()), + Arc::new(create_test_metadata()), ) .await; assert!(res.is_ok()); @@ -214,7 +231,7 @@ mod tests { chunks: vec![pb::TraceChunk { priority: i8::MIN as i32, origin: "".to_string(), - spans: vec![create_test_span(11, 222, 333, start, true)], + spans: vec![create_test_gcp_span(11, 222, 333, start, true)], tags: HashMap::new(), dropped_trace: false, }], @@ -230,7 +247,6 @@ mod tests { } else { None }; - assert_eq!(expected_tracer_payload, received_payload.unwrap()); } @@ -245,9 +261,9 @@ mod tests { let start = get_current_timestamp_nanos(); let json_trace = vec![ - create_test_json_span(11, 333, 222, start), - create_test_json_span(11, 222, 0, start), - create_test_json_span(11, 444, 333, start), + create_test_gcp_json_span(11, 333, 222, start), + create_test_gcp_json_span(11, 222, 0, start), + create_test_gcp_json_span(11, 444, 333, start), ]; let bytes = rmp_serde::to_vec(&vec![json_trace]).unwrap(); @@ -267,7 +283,7 @@ mod tests { Arc::new(create_test_config()), request, tx, - Arc::new(trace_utils::MiniAgentMetadata::default()), + Arc::new(create_test_metadata()), ) .await; assert!(res.is_ok()); @@ -286,9 +302,9 @@ mod tests { priority: i8::MIN as i32, origin: "".to_string(), spans: vec![ - create_test_span(11, 333, 222, start, false), - create_test_span(11, 222, 0, start, true), - create_test_span(11, 444, 333, start, false), + create_test_gcp_span(11, 333, 222, start, false), + create_test_gcp_span(11, 222, 0, start, true), + create_test_gcp_span(11, 444, 333, start, false), ], tags: HashMap::new(), dropped_trace: false, From 8f4dd0aa1bbb5e06eda3bc1919583477c2cbda86 Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Wed, 26 Feb 2025 18:24:22 -0500 Subject: [PATCH 56/64] ekump/APMSP-1756 add trace exporter integration tests (#897) - APMSP-1756 add initial integration tests for trace exporter - APMSP-1779 add span deserialization and Payload collection construction to send data integration tests - Return the offending key in the error message when decoding v04 msgpack spans and we encounter an invalid key - Fix confusing var name in dogstatsd client from PR #890 --- src/trace_processor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 14564e7..bdae880 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -194,7 +194,7 @@ mod tests { let start = get_current_timestamp_nanos(); - let json_span = create_test_json_span(11, 222, 333, start); + let json_span = create_test_json_span(11, 222, 333, start, false); let bytes = rmp_serde::to_vec(&vec![vec![json_span]]).unwrap(); let request = Request::builder() From cd118e03dc3734459ba4d38494a56758ae7a5b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:55:47 -0500 Subject: [PATCH 57/64] [trace-mini-agent] add trace aggregator (#907) * add `aggregator.rs` adds an aggregator which gives batches limited by intake payload size * update `TraceFlusher` to use the `TraceAggregator` * make `MiniAgent` use `TraceAggregator` on `TraceFlusher` * fmt + clippy * add license --- src/aggregator.rs | 171 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/mini_agent.rs | 5 +- src/trace_flusher.rs | 55 +++++++++----- 4 files changed, 209 insertions(+), 23 deletions(-) create mode 100644 src/aggregator.rs diff --git a/src/aggregator.rs b/src/aggregator.rs new file mode 100644 index 0000000..1f7a594 --- /dev/null +++ b/src/aggregator.rs @@ -0,0 +1,171 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use datadog_trace_utils::send_data::SendData; +use std::collections::VecDeque; + +/// Maximum content size per payload uncompressed in bytes, +/// that the Datadog Trace API accepts. The value is 3.2 MB. +/// +/// +pub const MAX_CONTENT_SIZE_BYTES: usize = (32 * 1_024 * 1_024) / 10; + +#[allow(clippy::module_name_repetitions)] +pub struct TraceAggregator { + queue: VecDeque, + max_content_size_bytes: usize, + buffer: Vec, +} + +impl Default for TraceAggregator { + fn default() -> Self { + TraceAggregator { + queue: VecDeque::new(), + max_content_size_bytes: MAX_CONTENT_SIZE_BYTES, + buffer: Vec::new(), + } + } +} + +impl TraceAggregator { + #[allow(dead_code)] + #[allow(clippy::must_use_candidate)] + pub fn new(max_content_size_bytes: usize) -> Self { + TraceAggregator { + queue: VecDeque::new(), + max_content_size_bytes, + buffer: Vec::new(), + } + } + + pub fn add(&mut self, p: SendData) { + self.queue.push_back(p); + } + + pub fn get_batch(&mut self) -> Vec { + let mut batch_size = 0; + + // Fill the batch + while batch_size < self.max_content_size_bytes { + if let Some(payload) = self.queue.pop_front() { + let payload_size = payload.len(); + + // Put stats back in the queue + if batch_size + payload_size > self.max_content_size_bytes { + self.queue.push_front(payload); + break; + } + batch_size += payload_size; + self.buffer.push(payload); + } else { + break; + } + } + + std::mem::take(&mut self.buffer) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use datadog_trace_utils::{ + trace_utils::TracerHeaderTags, tracer_payload::TracerPayloadCollection, + }; + use ddcommon::Endpoint; + + use super::*; + + #[test] + fn test_add() { + let mut aggregator = TraceAggregator::default(); + let tracer_header_tags = TracerHeaderTags { + lang: "lang", + lang_version: "lang_version", + lang_interpreter: "lang_interpreter", + lang_vendor: "lang_vendor", + tracer_version: "tracer_version", + container_id: "container_id", + client_computed_top_level: true, + client_computed_stats: true, + dropped_p0_traces: 0, + dropped_p0_spans: 0, + }; + let payload = SendData::new( + 1, + TracerPayloadCollection::V07(Vec::new()), + tracer_header_tags, + &Endpoint::from_slice("localhost"), + ); + + aggregator.add(payload.clone()); + assert_eq!(aggregator.queue.len(), 1); + assert_eq!(aggregator.queue[0].is_empty(), payload.is_empty()); + } + + #[test] + fn test_get_batch() { + let mut aggregator = TraceAggregator::default(); + let tracer_header_tags = TracerHeaderTags { + lang: "lang", + lang_version: "lang_version", + lang_interpreter: "lang_interpreter", + lang_vendor: "lang_vendor", + tracer_version: "tracer_version", + container_id: "container_id", + client_computed_top_level: true, + client_computed_stats: true, + dropped_p0_traces: 0, + dropped_p0_spans: 0, + }; + let payload = SendData::new( + 1, + TracerPayloadCollection::V07(Vec::new()), + tracer_header_tags, + &Endpoint::from_slice("localhost"), + ); + + aggregator.add(payload.clone()); + assert_eq!(aggregator.queue.len(), 1); + let batch = aggregator.get_batch(); + assert_eq!(batch.len(), 1); + } + + #[test] + fn test_get_batch_full_entries() { + let mut aggregator = TraceAggregator::new(2); + let tracer_header_tags = TracerHeaderTags { + lang: "lang", + lang_version: "lang_version", + lang_interpreter: "lang_interpreter", + lang_vendor: "lang_vendor", + tracer_version: "tracer_version", + container_id: "container_id", + client_computed_top_level: true, + client_computed_stats: true, + dropped_p0_traces: 0, + dropped_p0_spans: 0, + }; + let payload = SendData::new( + 1, + TracerPayloadCollection::V07(Vec::new()), + tracer_header_tags, + &Endpoint::from_slice("localhost"), + ); + + // Add 3 payloads + aggregator.add(payload.clone()); + aggregator.add(payload.clone()); + aggregator.add(payload.clone()); + + // The batch should only contain the first 2 payloads + let first_batch = aggregator.get_batch(); + assert_eq!(first_batch.len(), 2); + assert_eq!(aggregator.queue.len(), 1); + + // The second batch should only contain the last log + let second_batch = aggregator.get_batch(); + assert_eq!(second_batch.len(), 1); + assert_eq!(aggregator.queue.len(), 0); + } +} diff --git a/src/lib.rs b/src/lib.rs index ccbff24..ffdf908 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +pub mod aggregator; pub mod config; pub mod env_verifier; pub mod http_utils; diff --git a/src/mini_agent.rs b/src/mini_agent.rs index b6f548f..24afa05 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -62,12 +62,9 @@ impl MiniAgent { // start our trace flusher. receives trace payloads and handles buffering + deciding when to // flush to backend. let trace_flusher = self.trace_flusher.clone(); - let trace_config = self.config.clone(); tokio::spawn(async move { let trace_flusher = trace_flusher.clone(); - trace_flusher - .start_trace_flusher(trace_config.clone(), trace_rx) - .await; + trace_flusher.start_trace_flusher(trace_rx).await; }); // channels to send processed stats to our stats flusher. diff --git a/src/trace_flusher.rs b/src/trace_flusher.rs index 7004b33..a455cae 100644 --- a/src/trace_flusher.rs +++ b/src/trace_flusher.rs @@ -9,47 +9,64 @@ use tracing::{debug, error}; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; +use crate::aggregator::TraceAggregator; use crate::config::Config; #[async_trait] pub trait TraceFlusher { + fn new(aggregator: Arc>, config: Arc) -> Self + where + Self: Sized; /// Starts a trace flusher that listens for trace payloads sent to the tokio mpsc Receiver, /// implementing flushing logic that calls flush_traces. - async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver); - /// Flushes traces to the Datadog trace intake. - async fn flush_traces(&self, traces: Vec, config: Arc); + async fn start_trace_flusher(&self, mut rx: Receiver); + /// Given a `Vec`, a tracer payload, send it to the Datadog intake endpoint. + async fn send(&self, traces: Vec); + + /// Flushes traces by getting every available batch on the aggregator. + async fn flush(&self); } #[derive(Clone)] -pub struct ServerlessTraceFlusher {} +#[allow(clippy::module_name_repetitions)] +pub struct ServerlessTraceFlusher { + pub aggregator: Arc>, + pub config: Arc, +} #[async_trait] impl TraceFlusher for ServerlessTraceFlusher { - async fn start_trace_flusher(&self, config: Arc, mut rx: Receiver) { - let buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let buffer_producer = buffer.clone(); - let buffer_consumer = buffer.clone(); + fn new(aggregator: Arc>, config: Arc) -> Self { + ServerlessTraceFlusher { aggregator, config } + } + async fn start_trace_flusher(&self, mut rx: Receiver) { + let aggregator = Arc::clone(&self.aggregator); tokio::spawn(async move { while let Some(tracer_payload) = rx.recv().await { - let mut buffer = buffer_producer.lock().await; - buffer.push(tracer_payload); + let mut guard = aggregator.lock().await; + guard.add(tracer_payload); } }); loop { - tokio::time::sleep(time::Duration::from_secs(config.trace_flush_interval)).await; + tokio::time::sleep(time::Duration::from_secs(self.config.trace_flush_interval)).await; + self.flush().await; + } + } - let mut buffer = buffer_consumer.lock().await; - if !buffer.is_empty() { - self.flush_traces(buffer.to_vec(), config.clone()).await; - buffer.clear(); - } + async fn flush(&self) { + let mut guard = self.aggregator.lock().await; + + let mut traces = guard.get_batch(); + while !traces.is_empty() { + self.send(traces).await; + + traces = guard.get_batch(); } } - async fn flush_traces(&self, traces: Vec, config: Arc) { + async fn send(&self, traces: Vec) { if traces.is_empty() { return; } @@ -57,7 +74,7 @@ impl TraceFlusher for ServerlessTraceFlusher { for traces in trace_utils::coalesce_send_data(traces) { match traces - .send_proxy(config.proxy_url.as_deref()) + .send_proxy(self.config.proxy_url.as_deref()) .await .last_result { From bfd47b97849378f426e5d095fecdf71e55f8b04e Mon Sep 17 00:00:00 2001 From: Julio Gonzalez <107922352+hoolioh@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:18:14 +0100 Subject: [PATCH 58/64] Add v05 support in the trace exporter (#898) * Add v05 decoding. * Integrate v05 decoding/encoding in the trace exporter. * Use TraceChunkSpan as a common placeholder for v04 and v05 incoming payloads. * Add support for V05 payloads in TracerPayloadCollection. * Refactor send and send_deser_ser. * Prevent invalid modes combination in the builder in order to avoid panicking. * Improve error handling in collect_trace_chunks_function. * Enable conversion from v04 to v05. * Add integration tests for v05 format. * Solve PR comments. --- src/trace_processor.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index bdae880..f93cc12 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -95,7 +95,7 @@ impl TraceProcessor for ServerlessTraceProcessor { } }; - let payload = trace_utils::collect_trace_chunks( + let payload = match trace_utils::collect_trace_chunks( TraceCollection::V07(traces), &tracer_header_tags, &mut ChunkProcessor { @@ -103,7 +103,16 @@ impl TraceProcessor for ServerlessTraceProcessor { mini_agent_metadata: mini_agent_metadata.clone(), }, true, // In mini agent, we always send agentless - ); + false, + ) { + Ok(res) => res, + Err(err) => { + return log_and_create_traces_success_http_response( + &format!("Error processing trace chunks: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ) + } + }; let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); From 20f0c5f47d0104b7adbc4238dd37cfd848a1cf3c Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Fri, 14 Mar 2025 12:58:20 -0400 Subject: [PATCH 59/64] ekump/APMSP-1827 add warnings for panics (#915) Add clippy warnings and allows for panic macros to most crates Add an extension crate for mutex in ddcommon to isolate the unwrap to one location in code to avoid the allow annotations. We aren't going to stop unwrapping mutexes anytime soon. Replace use of lazy_static with OnceLock for ddcommon, ddtelemetry, live-debugger, tools, sidecar --- src/config.rs | 1 + src/lib.rs | 6 ++++++ src/stats_flusher.rs | 1 + 3 files changed, 8 insertions(+) diff --git a/src/config.rs b/src/config.rs index 4bb943e..0ea4050 100644 --- a/src/config.rs +++ b/src/config.rs @@ -69,6 +69,7 @@ impl Config { ) })?; + #[allow(clippy::unwrap_used)] Ok(Config { app_name: Some(app_name), env_type, diff --git a/src/lib.rs b/src/lib.rs index ffdf908..165c263 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,12 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::todo))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + pub mod aggregator; pub mod config; pub mod env_verifier; diff --git a/src/stats_flusher.rs b/src/stats_flusher.rs index ac8d59a..f2f6547 100644 --- a/src/stats_flusher.rs +++ b/src/stats_flusher.rs @@ -75,6 +75,7 @@ impl StatsFlusher for ServerlessStatsFlusher { } }; + #[allow(clippy::unwrap_used)] match stats_utils::send_stats_payload( serialized_stats_payload, &config.trace_stats_intake, From fdd0c1963e47addeb93e80e2025e1c383be5d6aa Mon Sep 17 00:00:00 2001 From: paullegranddc <82819397+paullegranddc@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:30:12 +0100 Subject: [PATCH 60/64] refactor(trace-utils): make flate2 and and hyper-proxy dependencies optionnal (#929) # What does this PR do? * Add a new feature "mini_agent" on trace_utils and do not use it bydefault. This is going to be used to progressively separate code used by only the mini agent from common code= * Hide flate2 usage under the compression flag * Hide hyper-proxy dependency under the proxy feature flag --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 381691f..5d2389f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" ddcommon = { path = "../ddcommon" } datadog-trace-protobuf = { path = "../trace-protobuf" } -datadog-trace-utils = { path = "../trace-utils", features = ["proxy"] } +datadog-trace-utils = { path = "../trace-utils", features = ["mini_agent"] } datadog-trace-normalization = { path = "../trace-normalization" } datadog-trace-obfuscation = { path = "../trace-obfuscation" } From db2fe688680cb6e53e2269b6ac8962bb4a30dee2 Mon Sep 17 00:00:00 2001 From: paullegranddc <82819397+paullegranddc@users.noreply.github.com> Date: Fri, 21 Mar 2025 17:17:10 +0100 Subject: [PATCH 61/64] chore: hyper 1.x migration (#946) # What does this PR do? Update all direct dependencies from hyper 0.14 to hyper 1.x The path of migration is not always obvious: * The hyper::Client struct was dropped * Migration: use hyper_utils::client::legacy * hyper::Body was dropped in favor of a trait that people are free to implement * Migration: This one is hard to fix. hyper::Body was a unifying struct over multiple behaviors (incoming and outgoing requests) (empty body, single body, streaming body) To match the previous behaviour I implemented hyper_migration::Body, an enum abstracting over all listed uses cases. This type is used everywhere to unify the Body types used in libdatadog * hyper::Server was dropped in favor of a simpler, connection oriented server * Migration: implement the accept spawn responder task loop ourselves. This is one of the things where I am the less sure we keep previous behaviour. Looking at hyper 0.14 code I don't think we are missing any part with the current migrated code in the mini-agent but without integration tests it's hard to be sure. * Because of the migration, hyper_util::Error is less descriptive than hyper::Error, which makes migration of the TraceExporterError not easy. We try to unwrap and find the source error as much as we can to preserve the behavior, and use anyhow to store the source error * hyper_proxy use in trace-util for the mini-agent has not migrated to hyper 1 (and is in general not maintained) * Migration: Use `hyper-http-proxy` a fork that did the update. We could also vendor the code because it's pretty small We're not completely free of hyper 0.14 though sadly because httpmock, a dev-dependency of the data-pipeline crate has not migrated yet to hyper 1. --- Cargo.toml | 5 ++- src/env_verifier.rs | 72 ++++++++++++++++++--------------- src/http_utils.rs | 24 ++++++----- src/mini_agent.rs | 91 ++++++++++++++++++++++++++++-------------- src/stats_processor.rs | 11 ++--- src/trace_processor.rs | 17 ++++---- 6 files changed, 132 insertions(+), 88 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5d2389f..f3f513f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,10 @@ bench = false [dependencies] anyhow = "1.0" -hyper = { version = "0.14", default-features = false, features = ["server", "backports", "deprecated"] } +hyper = { version = "1.6", features = ["http1", "client", "server"] } +hyper-util = {version = "0.1", features = ["service"] } +tower = { version = "0.5.2", features = ["util"] } +http-body-util = "0.1" tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" tracing = { version = "0.1", default-features = false } diff --git a/src/env_verifier.rs b/src/env_verifier.rs index f06f30d..ca0b137 100644 --- a/src/env_verifier.rs +++ b/src/env_verifier.rs @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; -use hyper::body::HttpBody; -use hyper::{Body, Client, Method, Request, Response}; +use ddcommon::hyper_migration; +use http_body_util::BodyExt; +use hyper::{Method, Request}; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -177,23 +178,24 @@ fn get_region_from_gcp_region_string(str: String) -> String { /// tests #[async_trait] pub(crate) trait GoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result>; + async fn get_metadata(&self) -> anyhow::Result; } + struct GoogleMetadataClientWrapper {} #[async_trait] impl GoogleMetadataClient for GoogleMetadataClientWrapper { - async fn get_metadata(&self) -> anyhow::Result> { + async fn get_metadata(&self) -> anyhow::Result { let req = Request::builder() .method(Method::POST) .uri(GCP_METADATA_URL) .header("Metadata-Flavor", "Google") - .body(Body::empty()) + .body(hyper_migration::Body::empty()) .map_err(|err| anyhow::anyhow!(err.to_string()))?; - let client = Client::new(); + let client = hyper_migration::new_default_client(); match client.request(req).await { - Ok(res) => Ok(res), + Ok(res) => Ok(hyper_migration::into_response(res)), Err(err) => anyhow::bail!(err.to_string()), } } @@ -233,7 +235,7 @@ async fn ensure_gcp_function_environment( Ok(gcp_metadata) } -async fn get_gcp_metadata_from_body(body: hyper::Body) -> anyhow::Result { +async fn get_gcp_metadata_from_body(body: hyper_migration::Body) -> anyhow::Result { let bytes = body.collect().await?.to_bytes(); let body_str = String::from_utf8(bytes.to_vec())?; let gcp_metadata: GCPMetadata = serde_json::from_str(&body_str)?; @@ -331,7 +333,8 @@ async fn ensure_azure_function_environment( mod tests { use async_trait::async_trait; use datadog_trace_utils::trace_utils; - use hyper::{Body, Response, StatusCode}; + use ddcommon::hyper_migration; + use hyper::{body::Bytes, Response, StatusCode}; use serde_json::json; use serial_test::serial; use std::{fs, path::Path, time::Duration}; @@ -351,7 +354,7 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result> { + async fn get_metadata(&self) -> anyhow::Result { anyhow::bail!("Random Error") } } @@ -371,11 +374,11 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result> { - Ok(Response::builder() - .status(StatusCode::OK) - .body(Body::empty()) - .unwrap()) + async fn get_metadata(&self) -> anyhow::Result { + Ok( + hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + .unwrap(), + ) } } let gmc = @@ -394,12 +397,13 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result> { - Ok(Response::builder() - .status(StatusCode::OK) - .header("Server", "Metadata Server NOT for Serverless") - .body(Body::empty()) - .unwrap()) + async fn get_metadata(&self) -> anyhow::Result { + Ok(hyper_migration::empty_response( + Response::builder() + .status(StatusCode::OK) + .header("Server", "Metadata Server NOT for Serverless"), + ) + .unwrap()) } } let gmc = @@ -418,11 +422,12 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result> { - Ok(Response::builder() - .status(StatusCode::OK) - .header("Server", "Metadata Server for Serverless") - .body(Body::from( + async fn get_metadata(&self) -> anyhow::Result { + Ok(hyper_migration::mock_response( + Response::builder() + .status(StatusCode::OK) + .header("Server", "Metadata Server for Serverless"), + Bytes::from( json!({ "instance": { "region": "projects/123123/regions/us-east1", @@ -432,8 +437,9 @@ mod tests { } }) .to_string(), - )) - .unwrap()) + ), + ) + .unwrap()) } } let gmc = @@ -459,13 +465,13 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result> { + async fn get_metadata(&self) -> anyhow::Result { // Sleep for 5 seconds to let the timeout trigger tokio::time::sleep(Duration::from_secs(5)).await; - Ok(Response::builder() - .status(StatusCode::OK) - .body(Body::empty()) - .unwrap()) + Ok( + hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + .unwrap(), + ) } } let gmc = diff --git a/src/http_utils.rs b/src/http_utils.rs index 01db15c..81bee93 100644 --- a/src/http_utils.rs +++ b/src/http_utils.rs @@ -1,10 +1,11 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use ddcommon::hyper_migration; use hyper::{ header, http::{self, HeaderMap}, - Body, Response, StatusCode, + Response, StatusCode, }; use serde_json::json; use tracing::{debug, error}; @@ -21,14 +22,16 @@ use tracing::{debug, error}; pub fn log_and_create_http_response( message: &str, status: StatusCode, -) -> http::Result> { +) -> http::Result> { if status.is_success() { debug!("{message}"); } else { error!("{message}"); } let body = json!({ "message": message }).to_string(); - Response::builder().status(status).body(Body::from(body)) + Response::builder() + .status(status) + .body(hyper_migration::Body::from(body)) } /// Does two things: @@ -45,10 +48,12 @@ pub fn log_and_create_http_response( pub fn log_and_create_traces_success_http_response( message: &str, status: StatusCode, -) -> http::Result> { +) -> http::Result { debug!("{message}"); let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); - Response::builder().status(status).body(Body::from(body)) + Response::builder() + .status(status) + .body(hyper_migration::Body::from(body)) } /// Takes a request's header map, and verifies that the "content-length" header is present, valid, @@ -60,7 +65,7 @@ pub fn verify_request_content_length( header_map: &HeaderMap, max_content_length: usize, error_message_prefix: &str, -) -> Option>> { +) -> Option> { let content_length_header = match header_map.get(header::CONTENT_LENGTH) { Some(res) => res, None => { @@ -99,11 +104,10 @@ pub fn verify_request_content_length( #[cfg(test)] mod tests { - use hyper::body::HttpBody; + use ddcommon::hyper_migration; + use http_body_util::BodyExt; use hyper::header; - use hyper::Body; use hyper::HeaderMap; - use hyper::Response; use hyper::StatusCode; use super::verify_request_content_length; @@ -114,7 +118,7 @@ mod tests { map } - async fn get_response_body_as_string(response: Response) -> String { + async fn get_response_body_as_string(response: hyper_migration::HttpResponse) -> String { let body = response.into_body(); let bytes = body.collect().await.unwrap().to_bytes(); String::from_utf8(bytes.into_iter().collect()).unwrap() diff --git a/src/mini_agent.rs b/src/mini_agent.rs index 24afa05..552c69d 100644 --- a/src/mini_agent.rs +++ b/src/mini_agent.rs @@ -1,10 +1,11 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use hyper::service::{make_service_fn, service_fn}; -use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; +use ddcommon::hyper_migration; +use hyper::service::service_fn; +use hyper::{http, Method, Response, StatusCode}; use serde_json::json; -use std::convert::Infallible; +use std::io; use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; @@ -88,7 +89,7 @@ impl MiniAgent { let stats_processor = self.stats_processor.clone(); let endpoint_config = self.config.clone(); - let make_svc = make_service_fn(move |_| { + let service = service_fn(move |req| { let trace_processor = trace_processor.clone(); let trace_tx = trace_tx.clone(); @@ -98,50 +99,78 @@ impl MiniAgent { let endpoint_config = endpoint_config.clone(); let mini_agent_metadata = Arc::clone(&mini_agent_metadata); - let service = service_fn(move |req| { - MiniAgent::trace_endpoint_handler( - endpoint_config.clone(), - req, - trace_processor.clone(), - trace_tx.clone(), - stats_processor.clone(), - stats_tx.clone(), - Arc::clone(&mini_agent_metadata), - ) - }); - - async move { Ok::<_, Infallible>(service) } + MiniAgent::trace_endpoint_handler( + endpoint_config.clone(), + req.map(hyper_migration::Body::incoming), + trace_processor.clone(), + trace_tx.clone(), + stats_processor.clone(), + stats_tx.clone(), + Arc::clone(&mini_agent_metadata), + ) }); let addr = SocketAddr::from(([127, 0, 0, 1], MINI_AGENT_PORT as u16)); - let server_builder = Server::try_bind(&addr)?; - - let server = server_builder.serve(make_svc); + let listener = tokio::net::TcpListener::bind(&addr).await?; debug!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); debug!( "Time taken start the Mini Agent: {} ms", now.elapsed().as_millis() ); - - // start hyper http server - if let Err(e) = server.await { - error!("Server error: {e}"); - return Err(e.into()); + let server = hyper::server::conn::http1::Builder::new(); + let mut joinset = tokio::task::JoinSet::new(); + loop { + let conn = tokio::select! { + con_res = listener.accept() => match con_res { + Err(e) + if matches!( + e.kind(), + io::ErrorKind::ConnectionAborted + | io::ErrorKind::ConnectionReset + | io::ErrorKind::ConnectionRefused + ) => + { + continue; + } + Err(e) => { + error!("Server error: {e}"); + return Err(e.into()); + } + Ok((conn, _)) => conn, + }, + finished = async { + match joinset.join_next().await { + Some(finished) => finished, + None => std::future::pending().await, + } + } => match finished { + Err(e) if e.is_panic() => { + std::panic::resume_unwind(e.into_panic()); + }, + Ok(()) | Err(_) => continue, + }, + }; + let conn = hyper_util::rt::TokioIo::new(conn); + let server = server.clone(); + let service = service.clone(); + joinset.spawn(async move { + if let Err(e) = server.serve_connection(conn, service).await { + error!("Connection error: {e}"); + } + }); } - - Ok(()) } async fn trace_endpoint_handler( config: Arc, - req: Request, + req: hyper_migration::HttpRequest, trace_processor: Arc, trace_tx: Sender, stats_processor: Arc, stats_tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result> { + ) -> http::Result { match (req.method(), req.uri().path()) { (&Method::PUT | &Method::POST, TRACE_ENDPOINT_PATH) => { match trace_processor @@ -179,7 +208,7 @@ impl MiniAgent { } } - fn info_handler(dd_dogstatsd_port: u16) -> http::Result> { + fn info_handler(dd_dogstatsd_port: u16) -> http::Result { let response_json = json!( { "endpoints": [ @@ -195,6 +224,6 @@ impl MiniAgent { ); Response::builder() .status(200) - .body(Body::from(response_json.to_string())) + .body(hyper_migration::Body::from(response_json.to_string())) } } diff --git a/src/stats_processor.rs b/src/stats_processor.rs index 28674eb..56bb061 100644 --- a/src/stats_processor.rs +++ b/src/stats_processor.rs @@ -5,7 +5,8 @@ use std::sync::Arc; use std::time::UNIX_EPOCH; use async_trait::async_trait; -use hyper::{http, Body, Request, Response, StatusCode}; +use ddcommon::hyper_migration; +use hyper::{http, StatusCode}; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -22,9 +23,9 @@ pub trait StatsProcessor { async fn process_stats( &self, config: Arc, - req: Request, + req: hyper_migration::HttpRequest, tx: Sender, - ) -> http::Result>; + ) -> http::Result; } #[derive(Clone)] @@ -35,9 +36,9 @@ impl StatsProcessor for ServerlessStatsProcessor { async fn process_stats( &self, config: Arc, - req: Request, + req: hyper_migration::HttpRequest, tx: Sender, - ) -> http::Result> { + ) -> http::Result { debug!("Received trace stats to process"); let (parts, body) = req.into_parts(); diff --git a/src/trace_processor.rs b/src/trace_processor.rs index f93cc12..232be1e 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -4,7 +4,8 @@ use std::sync::Arc; use async_trait::async_trait; -use hyper::{http, Body, Request, Response, StatusCode}; +use ddcommon::hyper_migration; +use hyper::{http, StatusCode}; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -26,10 +27,10 @@ pub trait TraceProcessor { async fn process_traces( &self, config: Arc, - req: Request, + req: hyper_migration::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result>; + ) -> http::Result; } struct ChunkProcessor { @@ -66,10 +67,10 @@ impl TraceProcessor for ServerlessTraceProcessor { async fn process_traces( &self, config: Arc, - req: Request, + req: hyper_migration::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result> { + ) -> http::Result { debug!("Received traces to process"); let (parts, body) = req.into_parts(); @@ -151,7 +152,7 @@ mod tests { use datadog_trace_utils::{ test_utils::create_test_json_span, trace_utils, tracer_payload::TracerPayloadCollection, }; - use ddcommon::Endpoint; + use ddcommon::{hyper_migration, Endpoint}; fn get_current_timestamp_nanos() -> i64 { UNIX_EPOCH.elapsed().unwrap().as_nanos() as i64 @@ -213,7 +214,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper::body::Body::from(bytes)) + .body(hyper_migration::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; @@ -283,7 +284,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper::body::Body::from(bytes)) + .body(hyper_migration::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; From 94f84ec7b574009bef6bf8d4aa1c474074a4367f Mon Sep 17 00:00:00 2001 From: paullegranddc <82819397+paullegranddc@users.noreply.github.com> Date: Wed, 16 Apr 2025 14:33:42 +0200 Subject: [PATCH 62/64] refactor: split collect_chunks into two methods (#1021) # What does this PR do? This refactor splits the logic in `collect_trace_chunks` between the trace exporter spans (v04 and v05) and the mini agent spans (pb::Spans). it completely removes usage of the `TraceCollection` struct from data-pipeline, and instead introduces the `TraceChunks` enum to differentiate between v04 and v05. Currently the way the code is structured makes replacing ByteString with the slice harder due to shared lifetime. Furthermore, the enums encodes two different codepaths, the spanBytes and span pb which never interact with each other. So having function that handle both span bytes and pb spans is pure complexity overhead. This refactor also removes a bunch of panics and lines of code that were here because to handle the "fake" pb spans and trace exporter spans overlap, which is practice never happens. Lastly, this remove the TracerParams struct. Every occurence of if was creating it, and invoking `TryInto` just after on it. So replacing it by a simple function is a lot less complex for the same feature set. # Motivation Prepare for using `SpanSlice<'a>` instead of `SpanBytes` in the trace exporter. --- src/trace_processor.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/trace_processor.rs b/src/trace_processor.rs index 232be1e..24a8c86 100644 --- a/src/trace_processor.rs +++ b/src/trace_processor.rs @@ -13,7 +13,7 @@ use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils::{self}; use datadog_trace_utils::trace_utils::{EnvironmentType, SendData}; -use datadog_trace_utils::tracer_payload::{TraceChunkProcessor, TraceCollection}; +use datadog_trace_utils::tracer_payload::TraceChunkProcessor; use crate::{ config::Config, @@ -96,15 +96,14 @@ impl TraceProcessor for ServerlessTraceProcessor { } }; - let payload = match trace_utils::collect_trace_chunks( - TraceCollection::V07(traces), + let payload = match trace_utils::collect_pb_trace_chunks( + traces, &tracer_header_tags, &mut ChunkProcessor { config: config.clone(), mini_agent_metadata: mini_agent_metadata.clone(), }, true, // In mini agent, we always send agentless - false, ) { Ok(res) => res, Err(err) => { From b59a13734db60b36300fcfa2528760d3b3d6b18c Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Wed, 16 Apr 2025 11:26:06 -0400 Subject: [PATCH 63/64] rename to trace-agent, update version, pin dependencies with libdatadog commit hash --- Cargo.lock | 2336 ++++++++++++++++- crates/trace-agent/Cargo.toml | 33 + .../src/aggregator.rs | 0 .../src/config.rs | 0 .../src/env_verifier.rs | 0 .../src/http_utils.rs | 0 .../src/lib.rs | 0 .../src/mini_agent.rs | 0 .../src/stats_flusher.rs | 0 .../src/stats_processor.rs | 0 .../src/trace_flusher.rs | 0 .../src/trace_processor.rs | 0 crates/trace-mini-agent/Cargo.toml | 35 - 13 files changed, 2273 insertions(+), 131 deletions(-) create mode 100644 crates/trace-agent/Cargo.toml rename crates/{trace-mini-agent => trace-agent}/src/aggregator.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/config.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/env_verifier.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/http_utils.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/lib.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/mini_agent.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/stats_flusher.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/stats_processor.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/trace_flusher.rs (100%) rename crates/{trace-mini-agent => trace-agent}/src/trace_processor.rs (100%) delete mode 100644 crates/trace-mini-agent/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index 9c9ad5a..e5769b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,12 +38,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -54,6 +78,177 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.3.1", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 0.38.44", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.4.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-object-pool" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333c456b97c3f2d50604e8b2624253b7f787208cb72eb75e64b0ad11b221652c" +dependencies = [ + "async-std", +] + +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel 2.3.1", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.4.0", + "futures-lite", + "rustix 0.38.44", + "tracing", +] + +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 0.38.44", + "signal-hook-registry", + "slab", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-std" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "730294c1c08c2e0f85759590518f6333f0d5a0a766a27d519c1b244c3dfd8a24" +dependencies = [ + "async-attributes", + "async-channel 1.9.0", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.88" @@ -62,7 +257,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -77,6 +272,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -92,21 +310,76 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "basic-cookies" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67bd8fd42c16bdb08688243dc5f0cc117a3ca9efeeaba3a345a18a6159ad96f7" +dependencies = [ + "lalrpop", + "lalrpop-util", + "regex", +] + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.100", + "which", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec 0.6.3", +] + [[package]] name = "bit-set" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit-vec" version = "0.8.0" @@ -119,6 +392,78 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel 2.3.1", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bollard" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41711ad46fda47cd701f6908e59d1bd6b9a2b7464c0d0aeab95c6d37096ff8a" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.3.1", + "http-body-util", + "hyper 1.6.0", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls", + "rustls-native-certs 0.7.3", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.45.0-rc.26.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7c5415e3a6bc6d3e99eff6268e488fd4ee25e7b28c10f08fa6760bd9de16e4" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -137,6 +482,38 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "cc" version = "1.2.16" @@ -148,6 +525,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -160,19 +546,307 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const_format" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.100", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + [[package]] name = "datadog-protos" version = "0.1.0" source = "git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222#c89b58e5784b985819baf11f13f7d35876741222" dependencies = [ "bytes", - "prost", + "prost 0.13.5", "protobuf", "protobuf-codegen", "tonic", "tonic-build", ] +[[package]] +name = "datadog-trace-normalization" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "anyhow", + "datadog-trace-protobuf", +] + +[[package]] +name = "datadog-trace-obfuscation" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "anyhow", + "datadog-trace-protobuf", + "datadog-trace-utils", + "ddcommon", + "log", + "percent-encoding", + "regex", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "datadog-trace-protobuf" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "prost 0.11.9", + "serde", + "serde_bytes", +] + +[[package]] +name = "datadog-trace-utils" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "anyhow", + "bytes", + "cargo-platform", + "cargo_metadata", + "datadog-trace-normalization", + "datadog-trace-protobuf", + "ddcommon", + "flate2", + "futures", + "http-body-util", + "httpmock", + "hyper 1.6.0", + "hyper-http-proxy", + "log", + "prost 0.11.9", + "rand 0.8.5", + "rmp", + "rmp-serde", + "rmpv", + "serde", + "serde_json", + "testcontainers", + "tinybytes", + "tokio", + "urlencoding", + "zstd", +] + +[[package]] +name = "ddcommon" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "anyhow", + "cc", + "const_format", + "futures", + "futures-core", + "futures-util", + "hex", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls", + "hyper-util", + "libc", + "log", + "memfd", + "nix", + "pin-project", + "rand 0.8.5", + "regex", + "rmp", + "rmp-serde", + "rustls", + "rustls-native-certs 0.8.1", + "serde", + "static_assertions", + "tokio", + "tokio-rustls", + "tower-service", + "windows-sys 0.52.0", +] + [[package]] name = "ddsketch-agent" version = "0.1.0" @@ -184,6 +858,16 @@ dependencies = [ "smallvec", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -201,10 +885,62 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -213,7 +949,18 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", +] + +[[package]] +name = "docker_credential" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", ] [[package]] @@ -224,7 +971,7 @@ dependencies = [ "ddsketch-agent", "derive_more", "fnv", - "hashbrown", + "hashbrown 0.15.2", "mockito", "proptest", "protobuf", @@ -241,12 +988,58 @@ dependencies = [ "zstd", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "duplicate" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a4be4cd710e92098de6ad258e6e7c24af11c29c5142f3c6f2a545652480ff8" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "ena" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +dependencies = [ + "log", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -263,18 +1056,61 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.0", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -299,6 +1135,27 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -306,6 +1163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -314,6 +1172,47 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-lite" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -332,10 +1231,26 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", ] [[package]] @@ -371,6 +1286,24 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.8" @@ -382,33 +1315,154 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", - "indexmap", + "http 1.3.1", + "indexmap 2.8.0", "slab", "tokio", - "tokio-util", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "headers" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +dependencies = [ + "base64 0.21.7", + "bytes", + "headers-core", + "http 1.3.1", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" +dependencies = [ + "http 1.3.1", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hickory-proto" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.8.5", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand 0.8.5", + "resolv-conf", + "smallvec", + "thiserror 1.0.69", + "tokio", "tracing", ] [[package]] -name = "hashbrown" -version = "0.15.2" +name = "home" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] [[package]] -name = "heck" -version = "0.5.0" +name = "hostname" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] [[package]] -name = "home" -version = "0.5.11" +name = "http" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "windows-sys 0.59.0", + "bytes", + "fnv", + "itoa", ] [[package]] @@ -422,6 +1476,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -429,7 +1494,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -440,8 +1505,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -457,6 +1522,57 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "httpmock" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ec9586ee0910472dec1a1f0f8acf52f0fdde93aea74d70d4a3107b4be0fd5b" +dependencies = [ + "assert-json-diff", + "async-object-pool", + "async-std", + "async-trait", + "base64 0.21.7", + "basic-cookies", + "crossbeam-utils", + "form_urlencoded", + "futures-util", + "hyper 0.14.32", + "lazy_static", + "levenshtein", + "log", + "regex", + "serde", + "serde_json", + "serde_regex", + "similar", + "tokio", + "url", +] + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -467,8 +1583,8 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -478,6 +1594,41 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-http-proxy" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ad4b0a1e37510028bc4ba81d0e38d239c39671b0f0ce9e02dfa93a8133f7c08" +dependencies = [ + "bytes", + "futures-util", + "headers", + "http 1.3.1", + "hyper 1.6.0", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.27.5" @@ -485,10 +1636,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", "rustls", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls", @@ -505,9 +1657,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -515,6 +1667,45 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -630,9 +1821,15 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -654,6 +1851,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.8.0" @@ -661,7 +1869,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", + "serde", +] + +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2", + "widestring", + "windows-sys 0.48.0", + "winreg", ] [[package]] @@ -670,6 +1891,33 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -704,18 +1952,96 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lalrpop" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" +dependencies = [ + "ascii-canvas", + "bit-set 0.5.3", + "ena", + "itertools 0.11.0", + "lalrpop-util", + "petgraph 0.6.5", + "pico-args", + "regex", + "regex-syntax 0.8.5", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", + "walkdir", +] + +[[package]] +name = "lalrpop-util" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +dependencies = [ + "regex-automata 0.4.9", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "levenshtein" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" + [[package]] name = "libc" version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -749,6 +2075,18 @@ name = "log" version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +dependencies = [ + "value-bag", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] [[package]] name = "matchers" @@ -765,12 +2103,27 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memfd" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +dependencies = [ + "rustix 0.38.44", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -800,10 +2153,10 @@ dependencies = [ "assert-json-diff", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "log", "rand 0.9.0", @@ -820,6 +2173,34 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -830,6 +2211,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.19" @@ -854,6 +2241,18 @@ version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "4.6.0" @@ -869,6 +2268,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" @@ -892,22 +2297,78 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax 0.8.5", + "structmeta", + "syn 2.0.100", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.8.0", +] + [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset", - "indexmap", + "fixedbitset 0.5.7", + "indexmap 2.8.0", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", ] +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + [[package]] name = "pin-project" version = "1.1.10" @@ -925,7 +2386,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -940,12 +2401,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 0.38.44", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -955,6 +2448,12 @@ dependencies = [ "zerocopy 0.8.23", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.2.31" @@ -962,7 +2461,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.100", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", ] [[package]] @@ -980,8 +2503,8 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ - "bit-set", - "bit-vec", + "bit-set 0.8.0", + "bit-vec 0.8.0", "bitflags", "lazy_static", "num-traits", @@ -994,6 +2517,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.13.5" @@ -1001,7 +2534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", ] [[package]] @@ -1010,20 +2543,33 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck", - "itertools", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.7.1", "prettyplease", - "prost", + "prost 0.13.5", "prost-types", "regex", - "syn", + "syn 2.0.100", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.13.5" @@ -1031,10 +2577,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1043,7 +2589,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] @@ -1080,7 +2626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.8.0", "log", "protobuf", "protobuf-support", @@ -1115,7 +2661,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror 2.0.12", @@ -1134,7 +2680,7 @@ dependencies = [ "getrandom 0.3.2", "rand 0.9.0", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -1251,6 +2797,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -1301,15 +2858,17 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", "h2", - "http", - "http-body", + "hickory-resolver", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-rustls", "hyper-util", "ipnet", @@ -1321,6 +2880,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", + "rustls-native-certs 0.8.1", "rustls-pemfile", "rustls-pki-types", "serde", @@ -1339,6 +2899,15 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "resolv-conf" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" +dependencies = [ + "hostname", +] + [[package]] name = "ring" version = "0.17.14" @@ -1353,12 +2922,50 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rmpv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58450723cd9ee93273ce44a20b6ec4efe17f8ed2e3631474387bfdecf18bb2a9" +dependencies = [ + "num-traits", + "rmp", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1397,6 +3004,7 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -1405,6 +3013,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -1429,6 +3062,7 @@ version = "0.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1458,6 +3092,24 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1465,47 +3117,188 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "serde" -version = "1.0.219" +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation 0.10.0", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.8.0", + "serde", "serde_derive", + "serde_json", + "serde_with_macros", + "time", ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "serde_with_macros" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ + "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] -name = "serde_json" -version = "1.0.140" +name = "serial_test" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot", + "serial_test_derive", ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "serial_test_derive" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", ] [[package]] @@ -1523,12 +3316,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -1560,12 +3368,70 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.100", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.100" @@ -1594,7 +3460,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1610,6 +3476,46 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "testcontainers" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef8374cea2c164699681ecc39316c3e1d953831a7a5721e36c7736d974e15fa" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "dirs", + "docker_credential", + "either", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "reqwest", + "serde", + "serde_json", + "serde_with", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1636,7 +3542,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1647,7 +3553,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1660,6 +3566,54 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinybytes" +version = "17.0.0" +source = "git+https://github.com/DataDog/libdatadog/?rev=3dab0bed2e144ce78c10a2378d1aff8fb5974f7d#3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" +dependencies = [ + "serde", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -1696,6 +3650,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.52.0", @@ -1709,7 +3664,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1753,14 +3708,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "tokio-stream", "tower-layer", "tower-service", @@ -1778,7 +3733,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1808,6 +3763,31 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +[[package]] +name = "trace-agent" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "datadog-trace-normalization", + "datadog-trace-obfuscation", + "datadog-trace-protobuf", + "datadog-trace-utils", + "ddcommon", + "duplicate", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "rmp-serde", + "serde", + "serde_json", + "serial_test", + "tempfile", + "tokio", + "tower", + "tracing", +] + [[package]] name = "tracing" version = "0.1.41" @@ -1827,7 +3807,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1887,7 +3867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1896,6 +3876,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unarray" version = "0.1.4" @@ -1929,8 +3915,15 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "ustr" version = "1.1.0" @@ -1961,6 +3954,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-bag" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" + [[package]] name = "version_check" version = "0.9.5" @@ -1976,6 +3975,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2022,7 +4031,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -2057,7 +4066,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2112,6 +4121,12 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "widestring" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" + [[package]] name = "winapi" version = "0.3.9" @@ -2128,12 +4143,56 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-link" version = "0.1.1" @@ -2147,7 +4206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -2169,6 +4228,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2187,6 +4264,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2219,6 +4311,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2231,6 +4329,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -2243,6 +4347,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2267,6 +4377,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -2279,6 +4395,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2291,6 +4413,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -2303,6 +4431,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2315,6 +4449,16 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -2356,7 +4500,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] @@ -2386,7 +4530,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2397,7 +4541,7 @@ checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2417,7 +4561,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] @@ -2446,7 +4590,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] diff --git a/crates/trace-agent/Cargo.toml b/crates/trace-agent/Cargo.toml new file mode 100644 index 0000000..a2b705e --- /dev/null +++ b/crates/trace-agent/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "trace-agent" +description = "A subset of the trace agent that is shipped alongside tracers in serverless environments" +version = "0.1.0" +license.workspace = true +edition.workspace = true + +[lib] +bench = false + +[dependencies] +anyhow = "1.0" +hyper = { version = "1.6", features = ["http1", "client", "server"] } +hyper-util = {version = "0.1", features = ["service"] } +tower = { version = "0.5.2", features = ["util"] } +http-body-util = "0.1" +tokio = { version = "1", features = ["macros", "rt-multi-thread"]} +async-trait = "0.1.64" +tracing = { version = "0.1", default-features = false } +serde = { version = "1.0.145", features = ["derive"] } +serde_json = "1.0" +ddcommon = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d", features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } + +[dev-dependencies] +rmp-serde = "1.1.1" +serial_test = "2.0.0" +duplicate = "0.4.1" +tempfile = "3.3.0" +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d", features=["test-utils"] } diff --git a/crates/trace-mini-agent/src/aggregator.rs b/crates/trace-agent/src/aggregator.rs similarity index 100% rename from crates/trace-mini-agent/src/aggregator.rs rename to crates/trace-agent/src/aggregator.rs diff --git a/crates/trace-mini-agent/src/config.rs b/crates/trace-agent/src/config.rs similarity index 100% rename from crates/trace-mini-agent/src/config.rs rename to crates/trace-agent/src/config.rs diff --git a/crates/trace-mini-agent/src/env_verifier.rs b/crates/trace-agent/src/env_verifier.rs similarity index 100% rename from crates/trace-mini-agent/src/env_verifier.rs rename to crates/trace-agent/src/env_verifier.rs diff --git a/crates/trace-mini-agent/src/http_utils.rs b/crates/trace-agent/src/http_utils.rs similarity index 100% rename from crates/trace-mini-agent/src/http_utils.rs rename to crates/trace-agent/src/http_utils.rs diff --git a/crates/trace-mini-agent/src/lib.rs b/crates/trace-agent/src/lib.rs similarity index 100% rename from crates/trace-mini-agent/src/lib.rs rename to crates/trace-agent/src/lib.rs diff --git a/crates/trace-mini-agent/src/mini_agent.rs b/crates/trace-agent/src/mini_agent.rs similarity index 100% rename from crates/trace-mini-agent/src/mini_agent.rs rename to crates/trace-agent/src/mini_agent.rs diff --git a/crates/trace-mini-agent/src/stats_flusher.rs b/crates/trace-agent/src/stats_flusher.rs similarity index 100% rename from crates/trace-mini-agent/src/stats_flusher.rs rename to crates/trace-agent/src/stats_flusher.rs diff --git a/crates/trace-mini-agent/src/stats_processor.rs b/crates/trace-agent/src/stats_processor.rs similarity index 100% rename from crates/trace-mini-agent/src/stats_processor.rs rename to crates/trace-agent/src/stats_processor.rs diff --git a/crates/trace-mini-agent/src/trace_flusher.rs b/crates/trace-agent/src/trace_flusher.rs similarity index 100% rename from crates/trace-mini-agent/src/trace_flusher.rs rename to crates/trace-agent/src/trace_flusher.rs diff --git a/crates/trace-mini-agent/src/trace_processor.rs b/crates/trace-agent/src/trace_processor.rs similarity index 100% rename from crates/trace-mini-agent/src/trace_processor.rs rename to crates/trace-agent/src/trace_processor.rs diff --git a/crates/trace-mini-agent/Cargo.toml b/crates/trace-mini-agent/Cargo.toml deleted file mode 100644 index f3f513f..0000000 --- a/crates/trace-mini-agent/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "datadog-trace-mini-agent" -description = "A subset of the trace agent that is shipped alongside tracers in a few serverless use cases (Google Cloud Functions, Azure Functions, and Azure Spring Apps)" -edition.workspace = true -version.workspace = true -rust-version.workspace = true -license.workspace = true -autobenches = false - -[lib] -bench = false - -[dependencies] -anyhow = "1.0" -hyper = { version = "1.6", features = ["http1", "client", "server"] } -hyper-util = {version = "0.1", features = ["service"] } -tower = { version = "0.5.2", features = ["util"] } -http-body-util = "0.1" -tokio = { version = "1", features = ["macros", "rt-multi-thread"]} -async-trait = "0.1.64" -tracing = { version = "0.1", default-features = false } -serde = { version = "1.0.145", features = ["derive"] } -serde_json = "1.0" -ddcommon = { path = "../ddcommon" } -datadog-trace-protobuf = { path = "../trace-protobuf" } -datadog-trace-utils = { path = "../trace-utils", features = ["mini_agent"] } -datadog-trace-normalization = { path = "../trace-normalization" } -datadog-trace-obfuscation = { path = "../trace-obfuscation" } - -[dev-dependencies] -rmp-serde = "1.1.1" -serial_test = "2.0.0" -duplicate = "0.4.1" -tempfile = "3.3.0" -datadog-trace-utils = { path = "../trace-utils", features=["test-utils"] } From 246c9913c49516f24027a1f74170b8faa25e8dda Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 18 Apr 2025 14:54:58 -0400 Subject: [PATCH 64/64] remove description --- crates/trace-agent/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/trace-agent/Cargo.toml b/crates/trace-agent/Cargo.toml index a2b705e..c273a75 100644 --- a/crates/trace-agent/Cargo.toml +++ b/crates/trace-agent/Cargo.toml @@ -1,6 +1,5 @@ [package] name = "trace-agent" -description = "A subset of the trace agent that is shipped alongside tracers in serverless environments" version = "0.1.0" license.workspace = true edition.workspace = true