From c6b8c60e979b1adf765c684a77a5e279d1306090 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 20 Oct 2025 15:58:58 -0400 Subject: [PATCH 1/4] feat: Increase trace payload limit from 2 MiB to 5 MiB --- bottlecap/Cargo.lock | 2 ++ bottlecap/Cargo.toml | 1 + bottlecap/src/traces/trace_agent.rs | 6 +++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 340e0afb7..bd91694b3 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -568,6 +568,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-util", + "tower-http", "tracing", "tracing-core", "tracing-subscriber", @@ -3720,6 +3721,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", + "http-body-util", "iri-string", "pin-project-lite", "tower 0.5.2", diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 9a162d5f7..bc295522b 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -49,6 +49,7 @@ opentelemetry-semantic-conventions = { version = "0.30", features = ["semconv_ex rustls-native-certs = { version = "0.8.1", optional = true } axum = { version = "0.8.4", default-features = false, features = ["default"] } ustr = { version = "1.0.0", default-features = false } +tower-http = { version = "0.6.6", default-features = false, features = ["limit"] } # If you are adding or updating a datadog-owned code dependency, please ensure # that it has a clippy.toml rule for disallowing the reqwest::Client::builder # method in favor of our diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 03500894a..fa935d017 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -3,11 +3,12 @@ use axum::{ Router, - extract::{Request, State}, + extract::{DefaultBodyLimit, Request, State}, http::StatusCode, response::{IntoResponse, Response}, routing::{any, post}, }; +use tower_http::limit::RequestBodyLimitLayer; use serde_json::json; use std::collections::VecDeque; use std::net::SocketAddr; @@ -72,6 +73,7 @@ const INSTRUMENTATION_INTAKE_PATH: &str = "/api/v2/apmtelemetry"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const STATS_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; +pub const TRACE_REQUEST_BODY_LIMIT: usize = 5 * 1024 * 1024; pub const MAX_CONTENT_LENGTH: usize = 10 * 1024 * 1024; const LAMBDA_LOAD_SPAN: &str = "aws.lambda.load"; @@ -231,6 +233,8 @@ impl TraceAgent { V5_TRACE_ENDPOINT_PATH, post(Self::v05_traces).put(Self::v05_traces), ) + .layer(DefaultBodyLimit::disable()) + .layer(RequestBodyLimitLayer::new(TRACE_REQUEST_BODY_LIMIT)) .with_state(trace_state); let stats_router = Router::new() From a13eebf4f86f3a26668e4844fc4aa092dc793d3c Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 21 Oct 2025 15:48:23 -0400 Subject: [PATCH 2/4] Change to 50 MiB --- bottlecap/src/traces/trace_agent.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index fa935d017..037a312ad 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -8,7 +8,6 @@ use axum::{ response::{IntoResponse, Response}, routing::{any, post}, }; -use tower_http::limit::RequestBodyLimitLayer; use serde_json::json; use std::collections::VecDeque; use std::net::SocketAddr; @@ -19,6 +18,7 @@ use tokio::sync::{ mpsc::{self, Receiver, Sender}, }; use tokio_util::sync::CancellationToken; +use tower_http::limit::RequestBodyLimitLayer; use tracing::{debug, error}; use crate::traces::trace_processor::SendingTraceProcessor; @@ -73,8 +73,9 @@ const INSTRUMENTATION_INTAKE_PATH: &str = "/api/v2/apmtelemetry"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const STATS_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; -pub const TRACE_REQUEST_BODY_LIMIT: usize = 5 * 1024 * 1024; -pub const MAX_CONTENT_LENGTH: usize = 10 * 1024 * 1024; +pub const TRACE_REQUEST_BODY_LIMIT: usize = 50 * 1024 * 1024; +pub const DEFAULT_REQUEST_BODY_LIMIT: usize = 2 * 1024 * 1024; +pub const MAX_CONTENT_LENGTH: usize = 50 * 1024 * 1024; const LAMBDA_LOAD_SPAN: &str = "aws.lambda.load"; #[derive(Clone)] @@ -233,12 +234,12 @@ impl TraceAgent { V5_TRACE_ENDPOINT_PATH, post(Self::v05_traces).put(Self::v05_traces), ) - .layer(DefaultBodyLimit::disable()) .layer(RequestBodyLimitLayer::new(TRACE_REQUEST_BODY_LIMIT)) .with_state(trace_state); let stats_router = Router::new() .route(STATS_ENDPOINT_PATH, post(Self::stats).put(Self::stats)) + .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)) .with_state(stats_state); let proxy_router = Router::new() @@ -258,9 +259,12 @@ impl TraceAgent { INSTRUMENTATION_ENDPOINT_PATH, post(Self::instrumentation_proxy), ) + .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)) .with_state(proxy_state); - let info_router = Router::new().route(INFO_ENDPOINT_PATH, any(Self::info)); + let info_router = Router::new() + .route(INFO_ENDPOINT_PATH, any(Self::info)) + .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)); Router::new() .merge(trace_router) @@ -268,6 +272,8 @@ impl TraceAgent { .merge(proxy_router) .merge(info_router) .fallback(handler_not_found) + // Disable the default body limit so we can use our own limit + .layer(DefaultBodyLimit::disable()) } async fn graceful_shutdown(shutdown_token: CancellationToken) { From d81731f4fb9c41dc45df8b79a23d99dde15fa4e9 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 21 Oct 2025 16:27:07 -0400 Subject: [PATCH 3/4] Remove limit on info_router --- bottlecap/src/traces/trace_agent.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 037a312ad..d2c764aaf 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -262,9 +262,7 @@ impl TraceAgent { .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)) .with_state(proxy_state); - let info_router = Router::new() - .route(INFO_ENDPOINT_PATH, any(Self::info)) - .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)); + let info_router = Router::new().route(INFO_ENDPOINT_PATH, any(Self::info)); Router::new() .merge(trace_router) From 2a66747c5f26f9dfdb93e4b2f4b1122e7b778369 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 21 Oct 2025 16:56:13 -0400 Subject: [PATCH 4/4] fmt --- bottlecap/src/traces/trace_agent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index d2c764aaf..27b0132fd 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -262,7 +262,7 @@ impl TraceAgent { .layer(RequestBodyLimitLayer::new(DEFAULT_REQUEST_BODY_LIMIT)) .with_state(proxy_state); - let info_router = Router::new().route(INFO_ENDPOINT_PATH, any(Self::info)); + let info_router = Router::new().route(INFO_ENDPOINT_PATH, any(Self::info)); Router::new() .merge(trace_router)