diff --git a/src/cli.rs b/src/cli.rs index cf22cf54..ed17d739 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -159,6 +159,13 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n #[clap(short = 'd', long)] pub download: bool, + /// During download, keep the raw encoding of the body. Intended for use with download or + /// to debug an encoded response body. + /// + /// For example, set Accept-Encoding: gzip and use --preserve-encoding to skip decompression. + #[clap(long)] + pub preserve_encoding: bool, + /// Resume an interrupted download. Requires --download and --output. #[clap( short = 'c', diff --git a/src/download.rs b/src/download.rs index c07fb6c8..3aa770cc 100644 --- a/src/download.rs +++ b/src/download.rs @@ -160,6 +160,7 @@ const UNCOLORED_SPINNER_TEMPLATE: &str = "{spinner} {bytes} {bytes_per_sec} {wid pub fn download_file( mut response: Response, + preserve_encoding: bool, file_name: Option, // If we fall back on taking the filename from the URL it has to be the // original URL, before redirects. That's less surprising and matches @@ -244,9 +245,13 @@ pub fn download_file( pb.reset_eta(); } + let compression_type = if !preserve_encoding { + get_compression_type(response.headers()) + } else { + None + }; match pb { Some(ref pb) => { - let compression_type = get_compression_type(response.headers()); copy_largebuf( &mut decompress(&mut pb.wrap_read(response), compression_type), &mut buffer, @@ -267,7 +272,6 @@ pub fn download_file( } } None => { - let compression_type = get_compression_type(response.headers()); copy_largebuf( &mut decompress(&mut response, compression_type), &mut buffer, diff --git a/src/main.rs b/src/main.rs index d5f7280a..cbca5f62 100644 --- a/src/main.rs +++ b/src/main.rs @@ -354,11 +354,25 @@ fn run(args: Cli) -> Result { let mut request = { let mut request_builder = client .request(method, url.clone()) - .header( + .header(USER_AGENT, get_user_agent()); + + if args.download { + if let Some(encoding) = headers.get(ACCEPT_ENCODING) { + if args.resume && encoding != HeaderValue::from_static("identity") { + return Err(anyhow!( + "Cannot use --continue with --download, when the encoding is not 'identity'" + )); + } + } else { + request_builder = + request_builder.header(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + } + } else { + request_builder = request_builder.header( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), - ) - .header(USER_AGENT, get_user_agent()); + ); + } if matches!( args.http_version, @@ -465,12 +479,6 @@ fn run(args: Cli) -> Result { request }; - if args.download { - request - .headers_mut() - .insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); - } - let buffer = Buffer::new( args.download, args.output.as_deref(), @@ -501,6 +509,7 @@ fn run(args: Cli) -> Result { printer.print_request_body(&mut request)?; } + let preserve_encoding = args.preserve_encoding; if !args.offline { let mut response = { let history_print = args.history_print.unwrap_or(print); @@ -515,6 +524,7 @@ fn run(args: Cli) -> Result { prev_response, response_charset, response_mime, + preserve_encoding, )?; printer.print_separator()?; } @@ -556,6 +566,7 @@ fn run(args: Cli) -> Result { if exit_code == 0 { download_file( response, + args.preserve_encoding, args.output, &url, resume, @@ -564,7 +575,12 @@ fn run(args: Cli) -> Result { )?; } } else if print.response_body { - printer.print_response_body(&mut response, response_charset, response_mime)?; + printer.print_response_body( + &mut response, + response_charset, + response_mime, + args.preserve_encoding, + )?; } } diff --git a/src/printer.rs b/src/printer.rs index 74b01ffd..5e0301e3 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -428,15 +428,33 @@ impl Printer { response: &mut Response, encoding: Option<&'static Encoding>, mime: Option<&str>, + preserve_encoding: bool, ) -> anyhow::Result<()> { let url = response.url().clone(); let content_type = mime.map_or_else(|| get_content_type(response.headers()), ContentType::from); let encoding = encoding.or_else(|| get_charset(response)); - let compression_type = get_compression_type(response.headers()); + let (may_decode, compression_type) = if !preserve_encoding { + let compression_type = get_compression_type(response.headers()); + (true, compression_type) + } else { + (false, None) + }; let mut body = decompress(response, compression_type); - if !self.buffer.is_terminal() { + if !may_decode { + // The user explicitly asked for preserving the encoding. We don't + // decode the body, even if it's text, so we can't write it to the terminal. + if self.buffer.is_terminal() { + self.buffer.print(BINARY_SUPPRESSOR)?; + } else if self.stream { + copy_largebuf(&mut body, &mut self.buffer, true)?; + } else { + let mut buf = Vec::new(); + body.read_to_end(&mut buf)?; + self.buffer.print(&buf)?; + } + } else if !self.buffer.is_terminal() { if (self.color || self.indent_json) && content_type.is_text() { // The user explicitly asked for formatting even though this is // going into a file, and the response is at least supposed to be diff --git a/tests/cli.rs b/tests/cli.rs index c853529a..436849cf 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -413,7 +413,7 @@ fn download() { } #[test] -fn accept_encoding_not_modifiable_in_download_mode() { +fn accept_encoding_identity_default_in_download_mode() { let server = server::http(|req| async move { assert_eq!(req.headers()["accept-encoding"], "identity"); hyper::Response::builder() @@ -424,11 +424,55 @@ fn accept_encoding_not_modifiable_in_download_mode() { let dir = tempdir().unwrap(); get_command() .current_dir(&dir) - .args([&server.base_url(), "--download", "accept-encoding:gzip"]) + .args([&server.base_url(), "--download"]) .assert() .success(); } +#[test] +fn accept_encoding_identity_in_resumable_download_mode() { + let server = server::http(|req| async move { + assert_eq!(req.headers()["accept-encoding"], "identity"); + hyper::Response::builder() + .body(r#"{"ids":[1,2,3]}"#.into()) + .unwrap() + }); + + let dir = tempdir().unwrap(); + get_command() + .current_dir(&dir) + .args([ + &server.base_url(), + "--download", + "--output", + "foo", + "--continue", + "accept-encoding:identity", + ]) + .assert() + .success(); +} + +#[test] +fn accept_encoding_not_modifiable_in_resumable_download_mode() { + let dir = tempdir().unwrap(); + get_command() + .current_dir(&dir) + .args([ + ":", + "--download", + "--output", + "foo", + "--continue", + "accept-encoding:gzip", + ]) + .assert() + .failure() + .stderr(indoc! {r#" + xh: error: Cannot use --continue with --download, when the encoding is not 'identity' + "#}); +} + #[test] fn download_generated_filename() { let dir = tempdir().unwrap();