Skip to content

Turbopack: allow gzip compression on trace files #84685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: sokra/set-output-in-task-completed
Choose a base branch
from
Open

Turbopack: allow gzip compression on trace files #84685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
74cf16f
Turbopack: allow gzip compression on trace files, write to diagnostic…
sokra Oct 9, 2025
1ea5893
fixup
sokra Oct 9, 2025
cb7af2b
detect from magic numbers
sokra Oct 9, 2025
4129b0f
clippy
sokra Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ dhat = { version = "0.3.2" }
dunce = "1.0.3"
either = "1.9.0"
erased-serde = "0.4.5"
flate2 = "1.0.28"
futures = "0.3.31"
futures-util = "0.3.31"
futures-retry = "0.6.0"
Expand Down
3 changes: 2 additions & 1 deletion crates/napi/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@ ignored = [
]

[dependencies]
anyhow = "1.0.66"
anyhow = { workspace = true }
console-subscriber = { workspace = true, optional = true }
dhat = { workspace = true, optional = true }
either = { workspace = true }
flate2 = { workspace = true }
futures-util = { workspace = true }
owo-colors = { workspace = true }
napi = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions crates/napi/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ DEALINGS IN THE SOFTWARE.
//#![deny(clippy::all)]
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![feature(iter_intersperse)]

#[macro_use]
extern crate napi_derive;
Expand Down
66 changes: 48 additions & 18 deletions crates/napi/src/next_api/project.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration};

use anyhow::{Context, Result, anyhow, bail};
use flate2::write::GzEncoder;
use futures_util::TryFutureExt;
use napi::{
Env, JsFunction, JsObject, Status,
Expand Down Expand Up @@ -363,21 +364,33 @@ pub fn project_new(
}

if let Some(mut trace) = trace {
// Trace presets
match trace.as_str() {
"overview" | "1" => {
trace = TRACING_NEXT_OVERVIEW_TARGETS.join(",");
}
"next" => {
trace = TRACING_NEXT_TARGETS.join(",");
}
"turbopack" => {
trace = TRACING_NEXT_TURBOPACK_TARGETS.join(",");
}
"turbo-tasks" => {
trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(",");
}
_ => {}
trace = trace
.split(",")
.map(|item| {
// Trace presets
match item {
"overview" | "1" => Cow::Owned(TRACING_NEXT_OVERVIEW_TARGETS.join(",")),
"next" => Cow::Owned(TRACING_NEXT_TARGETS.join(",")),
"turbopack" => Cow::Owned(TRACING_NEXT_TURBOPACK_TARGETS.join(",")),
"turbo-tasks" => Cow::Owned(TRACING_NEXT_TURBO_TASKS_TARGETS.join(",")),
_ => Cow::Borrowed(item),
}
})
.intersperse_with(|| Cow::Borrowed(","))
.collect::<String>();

enum Compression {
None,
GzipFast,
GzipBest,
}
let mut compress = Compression::None;
if trace.ends_with(",gz") {
trace.drain(trace.len() - 3..);
compress = Compression::GzipFast;
} else if trace.ends_with(",gz-best") {
trace.drain(trace.len() - 8..);
compress = Compression::GzipBest;
Comment on lines +388 to +393
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compression flags (gz or gz-best) are only detected when they appear at the very end of the trace string. If specified alone or in the middle of multiple presets, they are silently ignored and incorrectly treated as tracing filter directives.

View Details
📝 Patch Details 
diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs
index c32e785aaa..0d91923d90 100644
--- a/crates/napi/src/next_api/project.rs
+++ b/crates/napi/src/next_api/project.rs
@@ -364,35 +364,42 @@ pub fn project_new(
     }
 
     if let Some(mut trace) = trace {
+        enum Compression {
+            None,
+            GzipFast,
+            GzipBest,
+        }
+        let mut compress = Compression::None;
+
+        // Detect and remove compression flags before processing trace presets
         trace = trace
             .split(",")
-            .map(|item| {
-                // Trace presets
+            .filter_map(|item| {
+                // Check for compression flags
                 match item {
-                    "overview" | "1" => Cow::Owned(TRACING_NEXT_OVERVIEW_TARGETS.join(",")),
-                    "next" => Cow::Owned(TRACING_NEXT_TARGETS.join(",")),
-                    "turbopack" => Cow::Owned(TRACING_NEXT_TURBOPACK_TARGETS.join(",")),
-                    "turbo-tasks" => Cow::Owned(TRACING_NEXT_TURBO_TASKS_TARGETS.join(",")),
-                    _ => Cow::Borrowed(item),
+                    "gz" => {
+                        compress = Compression::GzipFast;
+                        None // Remove from trace list
+                    }
+                    "gz-best" => {
+                        compress = Compression::GzipBest;
+                        None // Remove from trace list
+                    }
+                    _ => {
+                        // Expand trace presets
+                        match item {
+                            "overview" | "1" => Some(Cow::Owned(TRACING_NEXT_OVERVIEW_TARGETS.join(","))),
+                            "next" => Some(Cow::Owned(TRACING_NEXT_TARGETS.join(","))),
+                            "turbopack" => Some(Cow::Owned(TRACING_NEXT_TURBOPACK_TARGETS.join(","))),
+                            "turbo-tasks" => Some(Cow::Owned(TRACING_NEXT_TURBO_TASKS_TARGETS.join(","))),
+                            _ => Some(Cow::Borrowed(item)),
+                        }
+                    }
                 }
             })
             .intersperse_with(|| Cow::Borrowed(","))
             .collect::<String>();
 
-        enum Compression {
-            None,
-            GzipFast,
-            GzipBest,
-        }
-        let mut compress = Compression::None;
-        if trace.ends_with(",gz") {
-            trace.drain(trace.len() - 3..);
-            compress = Compression::GzipFast;
-        } else if trace.ends_with(",gz-best") {
-            trace.drain(trace.len() - 8..);
-            compress = Compression::GzipBest;
-        }
-
         let subscriber = Registry::default();
 
         if cfg!(feature = "tokio-console") {

Analysis

Compression flags silently ignored unless at end of trace string

What fails: In crates/napi/src/next_api/project.rs, compression flags (gz or gz-best) in the NEXT_TURBOPACK_TRACING environment variable are only detected when appearing at the very end of the trace string, after all preset expansion.

How to reproduce:

# Standalone compression flag - fails
NEXT_TURBOPACK_TRACING=gz node server.js

# Compression in middle - fails  
NEXT_TURBOPACK_TRACING=overview,gz,turbopack node server.js

# Compression at end - works
NEXT_TURBOPACK_TRACING=overview,gz node server.js

Result: When compression flags appear standalone or in the middle, the code uses ends_with(",gz") check which fails, so compression is not enabled and the flag is passed to the tracing filter layer as an invalid target directive (no error shown).

Expected: Compression flags should be detected regardless of position in the comma-separated list, per the logic used for other preset flags like "overview" and "turbopack".

Root cause: The code expands presets and joins them with commas using intersperse_with (lines 367-381), then checks trace.ends_with(",gz") (lines 388-393). This only works if gz is literally at the end after joining.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we just compress by default? what is the tradeoff we are asking users to make?

}

let subscriber = Registry::default();
Expand All @@ -396,9 +409,26 @@ pub fn project_new(
std::fs::create_dir_all(&internal_dir)
.context("Unable to create .next directory")
.unwrap();
let trace_file = internal_dir.join("trace-turbopack");
let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
let (trace_writer, trace_writer_guard) = TraceWriter::new(trace_writer);
let trace_file;
let (trace_writer, trace_writer_guard) = match compress {
Compression::None => {
trace_file = internal_dir.join("trace-turbopack");
let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
TraceWriter::new(trace_writer)
}
Compression::GzipFast => {
trace_file = internal_dir.join("trace-turbopack");
let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::fast());
TraceWriter::new(trace_writer)
}
Compression::GzipBest => {
trace_file = internal_dir.join("trace-turbopack");
let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::best());
TraceWriter::new(trace_writer)
}
};
let subscriber = subscriber.with(RawTraceLayer::new(trace_writer));

exit.on_exit(async move {
Expand Down
2 changes: 1 addition & 1 deletion turbopack/crates/turbopack-trace-server/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ bench = false
[dependencies]
anyhow = { workspace = true, features = ["backtrace"] }
either = { workspace = true }
flate2 = { version = "1.0.28" }
flate2 = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] }
indexmap = { workspace = true, features = ["serde"] }
itertools = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions turbopack/crates/turbopack-trace-server/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#![feature(iter_intersperse)]
#![feature(box_patterns)]
#![feature(bufreader_peek)]

use std::{hash::BuildHasherDefault, path::PathBuf, sync::Arc};

Expand Down
1 change: 1 addition & 0 deletions turbopack/crates/turbopack-trace-server/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#![feature(iter_intersperse)]
#![feature(box_patterns)]
#![feature(bufreader_peek)]

use std::{hash::BuildHasherDefault, sync::Arc};

Expand Down
30 changes: 20 additions & 10 deletions turbopack/crates/turbopack-trace-server/src/reader/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ impl ObjectSafeTraceFormat for ErasedTraceFormat {

#[derive(Default)]
enum TraceFile {
Raw(File),
Raw(BufReader<File>),
Zstd(zstd::Decoder<'static, BufReader<File>>),
Gz(GzDecoder<BufReader<File>>),
#[default]
Expand Down Expand Up @@ -112,7 +112,7 @@ impl TraceFile {

fn size(&mut self) -> io::Result<u64> {
match self {
Self::Raw(file) => file.metadata().map(|m| m.len()),
Self::Raw(file) => file.get_ref().metadata().map(|m| m.len()),
Self::Zstd(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()),
Self::Gz(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()),
Self::Unloaded => unreachable!(),
Expand Down Expand Up @@ -145,13 +145,21 @@ impl TraceReader {

fn trace_file_from_file(&self, file: File) -> io::Result<TraceFile> {
let path = &self.path.to_string_lossy();
Ok(if path.ends_with(".zst") {
TraceFile::Zstd(zstd::Decoder::new(file)?)
} else if path.ends_with(".gz") {
TraceFile::Gz(GzDecoder::new(BufReader::new(file)))
} else {
TraceFile::Raw(file)
})
let mut file = BufReader::with_capacity(
// zstd max block size (1 << 17) + block header (3) + magic bytes (4)
(1 << 17) + 7,
file,
);
let magic_bytes = file.peek(4)?;
Ok(
if path.ends_with(".zst") || magic_bytes == [0x27, 0xb5, 0x2f, 0xfd] {
TraceFile::Zstd(zstd::Decoder::with_buffer(file)?)
} else if path.ends_with(".gz") || matches!(magic_bytes, [0x1f, 0x8b, _, _]) {
TraceFile::Gz(GzDecoder::new(file))
} else {
TraceFile::Raw(file)
},
)
}

fn try_read(&mut self) -> bool {
Expand Down Expand Up @@ -293,7 +301,9 @@ impl TraceReader {
}
}
Err(err) => {
if err.kind() == io::ErrorKind::UnexpectedEof {
if err.kind() == io::ErrorKind::UnexpectedEof
|| err.kind() == io::ErrorKind::InvalidInput
{
if let Some(value) = self.wait_for_more_data(
&mut file,
&mut initial_read,
Expand Down
Loading