From bcbb38a773275a1acc9e008659d9557758b07929 Mon Sep 17 00:00:00 2001 From: Jacob Trueb Date: Tue, 29 Nov 2022 17:13:45 -0600 Subject: [PATCH 1/5] Add request and keep encoding for download body flag --- src/cli.rs | 4 ++++ src/download.rs | 8 ++++++-- src/main.rs | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index d17ae81f..b33da735 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -159,6 +159,10 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n #[clap(short = 'd', long)] pub download: bool, + /// During download, request a content encoding and skip decompression. + #[clap(long)] + pub download_encoding: Option, + /// Resume an interrupted download. Requires --download and --output. #[clap( short = 'c', diff --git a/src/download.rs b/src/download.rs index c07fb6c8..8d2a2044 100644 --- a/src/download.rs +++ b/src/download.rs @@ -160,6 +160,7 @@ const UNCOLORED_SPINNER_TEMPLATE: &str = "{spinner} {bytes} {bytes_per_sec} {wid pub fn download_file( mut response: Response, + do_decompress: bool, file_name: Option, // If we fall back on taking the filename from the URL it has to be the // original URL, before redirects. That's less surprising and matches @@ -244,9 +245,13 @@ pub fn download_file( pb.reset_eta(); } + let compression_type = if do_decompress { + get_compression_type(response.headers()) + } else { + None + }; match pb { Some(ref pb) => { - let compression_type = get_compression_type(response.headers()); copy_largebuf( &mut decompress(&mut pb.wrap_read(response), compression_type), &mut buffer, @@ -267,7 +272,6 @@ pub fn download_file( } } None => { - let compression_type = get_compression_type(response.headers()); copy_largebuf( &mut decompress(&mut response, compression_type), &mut buffer, diff --git a/src/main.rs b/src/main.rs index 07fd53b2..3bec6d3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -465,9 +465,14 @@ fn run(args: Cli) -> Result { }; if args.download { - request - .headers_mut() - .insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + let encoding = args + .download_encoding + .clone() + .unwrap_or_else(|| "identity".to_string()); + request.headers_mut().insert( + ACCEPT_ENCODING, + HeaderValue::from_str(encoding.as_str()).unwrap(), + ); } let buffer = Buffer::new( @@ -555,6 +560,7 @@ fn run(args: Cli) -> Result { if exit_code == 0 { download_file( response, + args.download_encoding.is_none(), args.output, &args.url, resume, From f025ddfa6d518b0908a0828cde7406ee2b69e3c4 Mon Sep 17 00:00:00 2001 From: Jacob Trueb Date: Fri, 23 Dec 2022 12:46:19 -0600 Subject: [PATCH 2/5] Update download flag to keep raw encoding to preserve_encoding --- src/cli.rs | 8 +++++--- src/download.rs | 4 ++-- src/main.rs | 17 ++++++++--------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index b33da735..e514a555 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -159,9 +159,11 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n #[clap(short = 'd', long)] pub download: bool, - /// During download, request a content encoding and skip decompression. - #[clap(long)] - pub download_encoding: Option, + /// During download, keep the raw encoding of the body. Requires --download. + /// + /// For example, set Accept-Encoding: gzip and use --preserve-encoding to skip decompression. + #[clap(long, requires = "download")] + pub preserve_encoding: bool, /// Resume an interrupted download. Requires --download and --output. #[clap( diff --git a/src/download.rs b/src/download.rs index 8d2a2044..df15b5f7 100644 --- a/src/download.rs +++ b/src/download.rs @@ -160,7 +160,7 @@ const UNCOLORED_SPINNER_TEMPLATE: &str = "{spinner} {bytes} {bytes_per_sec} {wid pub fn download_file( mut response: Response, - do_decompress: bool, + preserve_encoding: bool, file_name: Option, // If we fall back on taking the filename from the URL it has to be the // original URL, before redirects. That's less surprising and matches @@ -245,7 +245,7 @@ pub fn download_file( pb.reset_eta(); } - let compression_type = if do_decompress { + let compression_type = if !preserve_encoding{ get_compression_type(response.headers()) } else { None diff --git a/src/main.rs b/src/main.rs index 3bec6d3c..4a43cf18 100644 --- a/src/main.rs +++ b/src/main.rs @@ -465,14 +465,13 @@ fn run(args: Cli) -> Result { }; if args.download { - let encoding = args - .download_encoding - .clone() - .unwrap_or_else(|| "identity".to_string()); - request.headers_mut().insert( - ACCEPT_ENCODING, - HeaderValue::from_str(encoding.as_str()).unwrap(), - ); + if let Some(encoding) = request.headers().get(ACCEPT_ENCODING) { + if args.resume && encoding != HeaderValue::from_static("identity") { + return Err(anyhow!("Cannot use --continue with --download, when the encoding is not 'identity'")); + } + } else { + request.headers_mut().insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + } } let buffer = Buffer::new( @@ -560,7 +559,7 @@ fn run(args: Cli) -> Result { if exit_code == 0 { download_file( response, - args.download_encoding.is_none(), + args.preserve_encoding, args.output, &args.url, resume, From dffabdc66104c65040718e69a34052eee3d09ad9 Mon Sep 17 00:00:00 2001 From: Jacob Trueb Date: Fri, 23 Dec 2022 13:04:56 -0600 Subject: [PATCH 3/5] Update formatting and test cases --- src/download.rs | 2 +- src/main.rs | 30 +++++++++++++----------- tests/cli.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/download.rs b/src/download.rs index df15b5f7..3aa770cc 100644 --- a/src/download.rs +++ b/src/download.rs @@ -245,7 +245,7 @@ pub fn download_file( pb.reset_eta(); } - let compression_type = if !preserve_encoding{ + let compression_type = if !preserve_encoding { get_compression_type(response.headers()) } else { None diff --git a/src/main.rs b/src/main.rs index 4a43cf18..609234cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -353,11 +353,25 @@ fn run(args: Cli) -> Result { let mut request = { let mut request_builder = client .request(method, args.url.clone()) - .header( + .header(USER_AGENT, get_user_agent()); + + if args.download { + if let Some(encoding) = headers.get(ACCEPT_ENCODING) { + if args.resume && encoding != HeaderValue::from_static("identity") { + return Err(anyhow!( + "Cannot use --continue with --download, when the encoding is not 'identity'" + )); + } + } else { + request_builder = + request_builder.header(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + } + } else { + request_builder = request_builder.header( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), - ) - .header(USER_AGENT, get_user_agent()); + ); + } if matches!( args.http_version, @@ -464,16 +478,6 @@ fn run(args: Cli) -> Result { request }; - if args.download { - if let Some(encoding) = request.headers().get(ACCEPT_ENCODING) { - if args.resume && encoding != HeaderValue::from_static("identity") { - return Err(anyhow!("Cannot use --continue with --download, when the encoding is not 'identity'")); - } - } else { - request.headers_mut().insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); - } - } - let buffer = Buffer::new( args.download, args.output.as_deref(), diff --git a/tests/cli.rs b/tests/cli.rs index 5d645510..3002f0c1 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -413,9 +413,9 @@ fn download() { } #[test] -fn accept_encoding_not_modifiable_in_download_mode() { +fn accept_encoding_modifiable_in_download_mode() { let server = server::http(|req| async move { - assert_eq!(req.headers()["accept-encoding"], "identity"); + assert_eq!(req.headers()["accept-encoding"], "gzip"); hyper::Response::builder() .body(r#"{"ids":[1,2,3]}"#.into()) .unwrap() @@ -429,6 +429,64 @@ fn accept_encoding_not_modifiable_in_download_mode() { .success(); } +#[test] +fn accept_encoding_identity_default_in_download_mode() { + let server = server::http(|req| async move { + assert_eq!(req.headers()["accept-encoding"], "identity"); + hyper::Response::builder() + .body(r#"{"ids":[1,2,3]}"#.into()) + .unwrap() + }); + + let dir = tempdir().unwrap(); + get_command() + .current_dir(&dir) + .args([&server.base_url(), "--download"]) + .assert() + .success(); +} + +#[test] +fn accept_encoding_identity_in_resumable_download_mode() { + let server = server::http(|req| async move { + assert_eq!(req.headers()["accept-encoding"], "identity"); + hyper::Response::builder() + .body(r#"{"ids":[1,2,3]}"#.into()) + .unwrap() + }); + + let dir = tempdir().unwrap(); + get_command() + .current_dir(&dir) + .args([ + &server.base_url(), + "--download", + "--output", + "foo", + "--continue", + "accept-encoding:identity", + ]) + .assert() + .success(); +} + +#[test] +fn accept_encoding_not_modifiable_in_resumable_download_mode() { + let dir = tempdir().unwrap(); + get_command() + .current_dir(&dir) + .args([ + ":", + "--download", + "--output", + "foo", + "--continue", + "accept-encoding:gzip", + ]) + .assert() + .failure(); +} + #[test] fn download_generated_filename() { let dir = tempdir().unwrap(); From eafb67663261097cb276cdca4ebca26fa16b52b3 Mon Sep 17 00:00:00 2001 From: Jacob Trueb Date: Tue, 3 Jan 2023 12:46:10 -0600 Subject: [PATCH 4/5] Update tests for --preserve-encoding flag --- tests/cli.rs | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tests/cli.rs b/tests/cli.rs index 3002f0c1..38bdc5f8 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -412,23 +412,6 @@ fn download() { assert_eq!(fs::read_to_string(&outfile).unwrap(), "file contents\n"); } -#[test] -fn accept_encoding_modifiable_in_download_mode() { - let server = server::http(|req| async move { - assert_eq!(req.headers()["accept-encoding"], "gzip"); - hyper::Response::builder() - .body(r#"{"ids":[1,2,3]}"#.into()) - .unwrap() - }); - - let dir = tempdir().unwrap(); - get_command() - .current_dir(&dir) - .args([&server.base_url(), "--download", "accept-encoding:gzip"]) - .assert() - .success(); -} - #[test] fn accept_encoding_identity_default_in_download_mode() { let server = server::http(|req| async move { @@ -484,7 +467,10 @@ fn accept_encoding_not_modifiable_in_resumable_download_mode() { "accept-encoding:gzip", ]) .assert() - .failure(); + .failure() + .stderr(indoc! {r#" + xh: error: Cannot use --continue with --download, when the encoding is not 'identity' + "#}); } #[test] From f6c7961cce4801d9daa5601cdb452b905989a0ca Mon Sep 17 00:00:00 2001 From: Jacob Trueb Date: Tue, 3 Jan 2023 13:21:02 -0600 Subject: [PATCH 5/5] Support --preserve-encoding without --download --- src/cli.rs | 5 +++-- src/main.rs | 9 ++++++++- src/printer.rs | 22 ++++++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index e514a555..bc32a63b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -159,10 +159,11 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n #[clap(short = 'd', long)] pub download: bool, - /// During download, keep the raw encoding of the body. Requires --download. + /// During download, keep the raw encoding of the body. Intended for use with download or + /// to debug an encoded response body. /// /// For example, set Accept-Encoding: gzip and use --preserve-encoding to skip decompression. - #[clap(long, requires = "download")] + #[clap(long)] pub preserve_encoding: bool, /// Resume an interrupted download. Requires --download and --output. diff --git a/src/main.rs b/src/main.rs index 609234cf..8f8c4a09 100644 --- a/src/main.rs +++ b/src/main.rs @@ -508,6 +508,7 @@ fn run(args: Cli) -> Result { printer.print_request_body(&mut request)?; } + let preserve_encoding = args.preserve_encoding; if !args.offline { let mut response = { let history_print = args.history_print.unwrap_or(print); @@ -522,6 +523,7 @@ fn run(args: Cli) -> Result { prev_response, response_charset, response_mime, + preserve_encoding, )?; printer.print_separator()?; } @@ -572,7 +574,12 @@ fn run(args: Cli) -> Result { )?; } } else if print.response_body { - printer.print_response_body(&mut response, response_charset, response_mime)?; + printer.print_response_body( + &mut response, + response_charset, + response_mime, + args.preserve_encoding, + )?; } } diff --git a/src/printer.rs b/src/printer.rs index 74b01ffd..5e0301e3 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -428,15 +428,33 @@ impl Printer { response: &mut Response, encoding: Option<&'static Encoding>, mime: Option<&str>, + preserve_encoding: bool, ) -> anyhow::Result<()> { let url = response.url().clone(); let content_type = mime.map_or_else(|| get_content_type(response.headers()), ContentType::from); let encoding = encoding.or_else(|| get_charset(response)); - let compression_type = get_compression_type(response.headers()); + let (may_decode, compression_type) = if !preserve_encoding { + let compression_type = get_compression_type(response.headers()); + (true, compression_type) + } else { + (false, None) + }; let mut body = decompress(response, compression_type); - if !self.buffer.is_terminal() { + if !may_decode { + // The user explicitly asked for preserving the encoding. We don't + // decode the body, even if it's text, so we can't write it to the terminal. + if self.buffer.is_terminal() { + self.buffer.print(BINARY_SUPPRESSOR)?; + } else if self.stream { + copy_largebuf(&mut body, &mut self.buffer, true)?; + } else { + let mut buf = Vec::new(); + body.read_to_end(&mut buf)?; + self.buffer.print(&buf)?; + } + } else if !self.buffer.is_terminal() { if (self.color || self.indent_json) && content_type.is_text() { // The user explicitly asked for formatting even though this is // going into a file, and the response is at least supposed to be