From 861cfb9e5e50b1ad9fc88c2f1918a0bfe68b6b79 Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 20:55:08 -0800 Subject: [PATCH 01/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/bench_run.rs | 5 +++++ src/configuration.rs | 13 +++++++++++-- src/http_bench_session.rs | 6 ++++++ src/metrics.rs | 5 +++++ src/prometheus_reporter.rs | 3 +++ 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/bench_run.rs b/src/bench_run.rs index 933c393..fd8bbc2 100644 --- a/src/bench_run.rs +++ b/src/bench_run.rs @@ -96,6 +96,7 @@ impl BenchRun { .expect("Unexpected LeakyBucket.acquire error"); let request_stats = bench_protocol_adapter.send_request(&client).await; + let fatal_error = request_stats.fatal_error; metrics_channel .try_send(request_stats) @@ -103,6 +104,10 @@ impl BenchRun { error!("Error sending metrics: {}", e); }) .unwrap_or_default(); + + if fatal_error { + break; + } } Ok(()) diff --git a/src/configuration.rs b/src/configuration.rs index 1ffdafc..f697fbc 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -68,6 +68,7 @@ impl BenchmarkConfig { (@arg IGNORE_CERT: --ignore_cert "Allow self signed certificates.") (@arg CONN_REUSE: --conn_reuse "If connections should be re-used") (@arg HTTP2_ONLY: --http2_only "Enforce HTTP/2 only") + (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") @@ -194,6 +195,7 @@ impl BenchmarkConfig { .ignore_cert(config.is_present("IGNORE_CERT")) .conn_reuse(config.is_present("CONN_REUSE")) .http2_only(config.is_present("HTTP2_ONLY")) + .stop_on_errors(BenchmarkConfig::parse_list(config, "STOP_ON_ERRORS")) .build() .expect("HttpClientConfigBuilder failed"), ) @@ -207,7 +209,7 @@ impl BenchmarkConfig { .collect(), ) .method(config.value_of("METHOD").unwrap_or("GET").to_string()) - .headers(BenchmarkConfig::get_multiple_values(config, "HEADER")) + .headers(BenchmarkConfig::parse_multiple_values(config, "HEADER")) .body(BenchmarkConfig::generate_body(config)) .build() .expect("HttpRequestBuilder failed"), @@ -262,7 +264,7 @@ impl BenchmarkConfig { buffer } - fn get_multiple_values(config: &ArgMatches, id: &str) -> Vec<(String, String)> { + fn parse_multiple_values(config: &ArgMatches, id: &str) -> Vec<(String, String)> { config .values_of(id) .map(|v| { @@ -278,6 +280,13 @@ impl BenchmarkConfig { .unwrap_or_else(Vec::new) } + fn parse_list(config: &ArgMatches, id: &str) -> Vec { + config + .values_of(id) + .map(|v| v.map(|s| s.to_string()).collect()) + .unwrap_or_else(Vec::new) + } + pub fn new_bench_session(&mut self) -> BenchSession { BenchSessionBuilder::default() .concurrency(self.concurrency) diff --git a/src/http_bench_session.rs b/src/http_bench_session.rs index 3fbb1e8..94bd321 100644 --- a/src/http_bench_session.rs +++ b/src/http_bench_session.rs @@ -34,6 +34,8 @@ pub struct HttpClientConfig { conn_reuse: bool, #[builder(default)] http2_only: bool, + #[builder(default)] + pub stop_on_errors: Vec, } #[derive(Builder, Deserialize, Clone, Debug)] @@ -126,6 +128,9 @@ impl BenchmarkProtocolAdapter for HttpBenchAdapter { Ok(r) => { let status = r.status().to_string(); let success = r.status().is_success(); + + let fatal_error = !success && self.config.stop_on_errors.contains(&status); + let mut stream = r.into_body(); let mut total_size = 0; while let Some(item) = stream.next().await { @@ -140,6 +145,7 @@ impl BenchmarkProtocolAdapter for HttpBenchAdapter { .status(status) .is_success(success) .duration(Instant::now().duration_since(start)) + .fatal_error(fatal_error) .build() .expect("RequestStatsBuilder failed") } diff --git a/src/metrics.rs b/src/metrics.rs index 6a722c4..e780d86 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -61,6 +61,7 @@ pub struct RequestStats { pub duration: Duration, #[builder(default = "None")] pub operation_name: Option, + pub fatal_error: bool, } impl BenchRunMetrics { @@ -392,6 +393,7 @@ mod tests { status: code, duration: Default::default(), operation_name: None, + fatal_error: false, }); } @@ -421,6 +423,7 @@ mod tests { status: "200 OK".to_string(), duration: Duration::from_micros(i), operation_name: None, + fatal_error: false, }); } @@ -452,6 +455,7 @@ mod tests { } else { Some("OperationB".to_string()) }, + fatal_error: false, }); } @@ -538,6 +542,7 @@ mod tests { status: "200 OK".to_string(), duration: Duration::from_micros(i), operation_name: None, + fatal_error: false, }); } diff --git a/src/prometheus_reporter.rs b/src/prometheus_reporter.rs index 095cb8a..538ebb0 100644 --- a/src/prometheus_reporter.rs +++ b/src/prometheus_reporter.rs @@ -372,6 +372,7 @@ mod test { .status(code) .is_success(success) .duration(Duration::from_micros(i as u64)) + .fatal_error(false) .build() .expect("RequestStatsBuilder failed"), ); @@ -478,6 +479,7 @@ mod test { .bytes_processed(i) .status(code) .is_success(success) + .fatal_error(false) .duration(Duration::from_micros(i as u64)) .build() .expect("RequestStatsBuilder failed"), @@ -594,6 +596,7 @@ mod test { status: "200 OK".to_string(), duration: Duration::from_micros(i), operation_name: None, + fatal_error: false, }); } From 8e145c02156fad1fc054bdca61bc17fa863c78f1 Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 21:15:35 -0800 Subject: [PATCH 02/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/http_bench_session.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http_bench_session.rs b/src/http_bench_session.rs index 94bd321..f11b1f0 100644 --- a/src/http_bench_session.rs +++ b/src/http_bench_session.rs @@ -157,6 +157,7 @@ impl BenchmarkProtocolAdapter for HttpBenchAdapter { .status(status) .is_success(false) .duration(Instant::now().duration_since(start)) + .fatal_error(false) .build() .expect("RequestStatsBuilder failed") } From 20dd1e46d17ffcdfe68182c909d2665c3a473ddb Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 21:30:16 -0800 Subject: [PATCH 03/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/http_bench_session.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/http_bench_session.rs b/src/http_bench_session.rs index f11b1f0..788f14e 100644 --- a/src/http_bench_session.rs +++ b/src/http_bench_session.rs @@ -170,9 +170,8 @@ impl HttpRequest { let method = Method::from_str(&self.method.clone()).expect("Method must be valid at this point"); - let mut request_builder = Request::builder() - .method(method) - .uri(&self.url[thread_rng().gen_range(0..self.url.len())].clone()); + let uri = &self.url[thread_rng().gen_range(0..self.url.len())]; + let mut request_builder = Request::builder().method(method).uri(uri.clone()); if !self.headers.is_empty() { for (key, value) in self.headers.iter() { @@ -190,6 +189,12 @@ impl HttpRequest { } else { request_builder .body(Body::empty()) + .map_err(|e| { + println!( + "Cannot create url {}, headers: {:?}. Error: {}", + uri, self.headers, e + ); + }) .expect("Error building Request") } } From f2d7632e5efd90e9bbcf166262f01bb81e58d90e Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 21:40:23 -0800 Subject: [PATCH 04/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/configuration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configuration.rs b/src/configuration.rs index f697fbc..a2127af 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -68,8 +68,8 @@ impl BenchmarkConfig { (@arg IGNORE_CERT: --ignore_cert "Allow self signed certificates.") (@arg CONN_REUSE: --conn_reuse "If connections should be re-used") (@arg HTTP2_ONLY: --http2_only "Enforce HTTP/2 only") - (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") + (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") From be71d30536a15748975d0ea4c941a0f4f9655b52 Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 21:43:50 -0800 Subject: [PATCH 05/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/configuration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configuration.rs b/src/configuration.rs index a2127af..f5b9dd8 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -69,9 +69,9 @@ impl BenchmarkConfig { (@arg CONN_REUSE: --conn_reuse "If connections should be re-used") (@arg HTTP2_ONLY: --http2_only "Enforce HTTP/2 only") (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") - (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") + (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") ) ).get_matches(); From d8d6dd17abe6b3dfe1de7bec9c10dc9489566623 Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 21:53:16 -0800 Subject: [PATCH 06/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/configuration.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/configuration.rs b/src/configuration.rs index f5b9dd8..bd7834a 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -71,7 +71,7 @@ impl BenchmarkConfig { (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") - (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") + (@arg STOP_ON_ERRORS: --error_stop -E +takes_value "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") ) ).get_matches(); @@ -281,10 +281,11 @@ impl BenchmarkConfig { } fn parse_list(config: &ArgMatches, id: &str) -> Vec { - config - .values_of(id) - .map(|v| v.map(|s| s.to_string()).collect()) - .unwrap_or_else(Vec::new) + if let Some(value) = config.value_of(id) { + value.split(',').map(|s| s.to_string()).collect() + } else { + vec![] + } } pub fn new_bench_session(&mut self) -> BenchSession { From 95176e81a3dd983e80615b74ea1532b084389ace Mon Sep 17 00:00:00 2001 From: xnuter Date: Tue, 8 Mar 2022 22:02:46 -0800 Subject: [PATCH 07/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/configuration.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/configuration.rs b/src/configuration.rs index bd7834a..9ea4c70 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -71,7 +71,6 @@ impl BenchmarkConfig { (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") - (@arg STOP_ON_ERRORS: --error_stop -E +takes_value "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") ) ).get_matches(); @@ -195,7 +194,7 @@ impl BenchmarkConfig { .ignore_cert(config.is_present("IGNORE_CERT")) .conn_reuse(config.is_present("CONN_REUSE")) .http2_only(config.is_present("HTTP2_ONLY")) - .stop_on_errors(BenchmarkConfig::parse_list(config, "STOP_ON_ERRORS")) + .stop_on_errors(vec!["403".to_string(), "401".to_string()]) .build() .expect("HttpClientConfigBuilder failed"), ) From c5ef4991530e00392c1c00e268a6e042e462dc85 Mon Sep 17 00:00:00 2001 From: xnuter Date: Wed, 9 Mar 2022 17:51:38 -0800 Subject: [PATCH 08/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- Cargo.toml | 2 +- src/configuration.rs | 14 +++++++++++--- src/http_bench_session.rs | 5 +++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9846db9..41fd095 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ Gauging performance of network services. Snapshot or continuous, supports Promet # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -clap = "2.33" +clap = "3.1.6" base64 = "0.13" derive_builder = "0.9" log = "0.4" diff --git a/src/configuration.rs b/src/configuration.rs index 9ea4c70..5735b1c 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -71,6 +71,7 @@ impl BenchmarkConfig { (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") (@arg METHOD: --method -M +takes_value "Method. By default GET") (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") + (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") ) ).get_matches(); @@ -92,6 +93,10 @@ impl BenchmarkConfig { .value_of("NUMBER_OF_REQUESTS") .map(|n| parse_num(n, "Illegal number for NUMBER_OF_REQUESTS")); + if duration.is_none() && number_of_requests.is_none() { + panic!("Either the number of requests or the test duration must be specified"); + } + let rate_ladder = if let Some(rate_max) = rate_max { let rate_per_second = rate_per_second.expect("RATE is required if RATE_MAX is specified"); @@ -194,7 +199,7 @@ impl BenchmarkConfig { .ignore_cert(config.is_present("IGNORE_CERT")) .conn_reuse(config.is_present("CONN_REUSE")) .http2_only(config.is_present("HTTP2_ONLY")) - .stop_on_errors(vec!["403".to_string(), "401".to_string()]) + .stop_on_errors(BenchmarkConfig::parse_list(config, "STOP_ON_ERRORS")) .build() .expect("HttpClientConfigBuilder failed"), ) @@ -279,9 +284,12 @@ impl BenchmarkConfig { .unwrap_or_else(Vec::new) } - fn parse_list(config: &ArgMatches, id: &str) -> Vec { + fn parse_list(config: &ArgMatches, id: &str) -> Vec { if let Some(value) = config.value_of(id) { - value.split(',').map(|s| s.to_string()).collect() + value + .split(',') + .map(|s| parse_num::(s, "Cannot parse error code")) + .collect() } else { vec![] } diff --git a/src/http_bench_session.rs b/src/http_bench_session.rs index 788f14e..980f5f1 100644 --- a/src/http_bench_session.rs +++ b/src/http_bench_session.rs @@ -35,7 +35,7 @@ pub struct HttpClientConfig { #[builder(default)] http2_only: bool, #[builder(default)] - pub stop_on_errors: Vec, + pub stop_on_errors: Vec, } #[derive(Builder, Deserialize, Clone, Debug)] @@ -129,7 +129,8 @@ impl BenchmarkProtocolAdapter for HttpBenchAdapter { let status = r.status().to_string(); let success = r.status().is_success(); - let fatal_error = !success && self.config.stop_on_errors.contains(&status); + let fatal_error = + !success && self.config.stop_on_errors.contains(&r.status().as_u16()); let mut stream = r.into_body(); let mut total_size = 0; From 9b9cde1c70f55a69b48d26c461fc60baea239789 Mon Sep 17 00:00:00 2001 From: xnuter Date: Thu, 10 Mar 2022 18:49:35 -0800 Subject: [PATCH 09/12] Exiting on errors. E.g. when an auth token expires, we want to refresh it. --- src/bench_run.rs | 6 +++++- src/configuration.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/bench_run.rs b/src/bench_run.rs index fd8bbc2..9bf5411 100644 --- a/src/bench_run.rs +++ b/src/bench_run.rs @@ -9,9 +9,12 @@ use crate::rate_limiter::RateLimiter; /// except according to those terms. use async_trait::async_trait; use log::error; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use tokio::sync::mpsc::Sender; +static STOP_ON_FATAL: AtomicBool = AtomicBool::new(false); + #[derive(Clone, Debug)] pub struct BenchRun { pub index: usize, @@ -89,7 +92,7 @@ impl BenchRun { ) -> Result<(), String> { let client = bench_protocol_adapter.build_client()?; - while self.has_more_work() { + while self.has_more_work() && !STOP_ON_FATAL.load(Ordering::Relaxed) { self.rate_limiter .acquire_one() .await @@ -106,6 +109,7 @@ impl BenchRun { .unwrap_or_default(); if fatal_error { + STOP_ON_FATAL.store(true, Ordering::Relaxed); break; } } diff --git a/src/configuration.rs b/src/configuration.rs index 5735b1c..6126a88 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -318,7 +318,7 @@ impl fmt::Display for BenchmarkConfig { pub fn parse_num(s: &str, error_msg: &str) -> F { s.parse() .map_err(|_| { - println!("{}", error_msg); + println!("{}. Non-number value: {}", error_msg, s); panic!("Cannot start"); }) .unwrap() From 4a1647f1b613151a0840332162d9a0957cf79650 Mon Sep 17 00:00:00 2001 From: xnuter Date: Sat, 12 Mar 2022 20:28:30 -0800 Subject: [PATCH 10/12] Upgrade `clap` --- Cargo.toml | 2 +- src/configuration.rs | 308 +++++++++++++++++++++++-------------------- 2 files changed, 166 insertions(+), 144 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 41fd095..aefca74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ Gauging performance of network services. Snapshot or continuous, supports Promet # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -clap = "3.1.6" +clap = { version = "3.1.6", features = ["derive"] } base64 = "0.13" derive_builder = "0.9" log = "0.4" diff --git a/src/configuration.rs b/src/configuration.rs index 6126a88..4111e08 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -10,12 +10,15 @@ use crate::http_bench_session::{ HttpBenchAdapter, HttpBenchAdapterBuilder, HttpClientConfigBuilder, HttpRequestBuilder, }; use crate::metrics::{DefaultConsoleReporter, ExternalMetricsServiceReporter}; -use clap::{clap_app, ArgMatches}; +use clap::Args; +use clap::Parser; +use clap::Subcommand; use core::fmt; use rand::Rng; use std::fs; use std::fs::File; use std::io::Read; +#[cfg(feature = "tls-boring")] use std::process::exit; use std::str::FromStr; use std::sync::Arc; @@ -42,56 +45,104 @@ pub struct BenchmarkConfig { pub reporters: Vec>, } +#[derive(Parser, Debug)] +// #[clap(name = "Performance Gauge")] +// #[clap(author = "Eugene Retunsky")] +// #[clap(version = "0.1.9")] +// #[clap(about = "A tool for gauging performance of network services", long_about = None)] +#[clap(author, version, about, long_about = None)] +#[clap(propagate_version = true)] +struct Cli { + /// Concurrent clients. Default `1`. + #[clap(short, long, default_value_t = 1)] + concurrency: usize, + /// Duration of the test. + #[clap(short, long)] + duration: Option, + /// Number of requests per client. + #[clap(short, long = "num_req")] + num_req: Option, + /// Test case name. Optional. Can be used for tagging metrics. + #[clap(short = 'N', long)] + name: Option, + /// Request rate per second. E.g. 100 or 0.1. By default no limit. + #[clap(short, long)] + rate: Option, + /// Rate increase step (until it reaches --rate_max). + #[clap(long = "rate_step")] + rate_step: Option, + /// Max rate per second. Requires --rate-step + #[clap(long = "rate_max")] + rate_max: Option, + /// takes_value "The number of iterations with the max rate. By default `1`. + #[clap(short, long = "max_iter", default_value_t = 1)] + max_iter: usize, + /// If it's a part of a continuous run. In this case metrics are not reset at the end to avoid saw-like plots. + #[clap(long)] + continuous: bool, + /// If you'd like to send metrics to Prometheus PushGateway, specify the server URL. E.g. 10.0.0.1:9091 + #[clap(long)] + prometheus: Option, + /// Prometheus Job (by default `pushgateway`) + #[clap(long = "prometheus_job")] + prometheus_job: Option, + #[clap(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + Http(HttpOptions), +} + +#[derive(Args, Debug)] +#[clap(about = "Run in HTTP(S) mode", long_about = None)] +#[clap(author, version, long_about = None)] +#[clap(propagate_version = true)] +struct HttpOptions { + /// Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing). + #[clap()] + target: Vec, + /// Headers in \"Name:Value\" form. Can be provided multiple times. + #[clap(short = 'H', long)] + header: Vec, + /// Method. By default GET. + #[clap(short = 'M', long)] + method: Option, + /// Stop immediately on error codes. E.g. `-E 401 -E 403` + #[clap(short = 'E', long = "error_stop")] + error_stop: Vec, + /// Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional. + #[clap(short = 'B', long)] + body: Option, + /// Allow self signed certificates. + #[clap(long = "ignore_cert")] + ignore_cert: bool, + /// If connections should be re-used. + #[clap(long = "conn_reuse")] + conn_reuse: bool, + /// Enforce HTTP/2 only. + #[clap(long = "http2_only")] + http2_only: bool, +} + impl BenchmarkConfig { pub fn from_command_line() -> io::Result { - let matches = clap_app!(myapp => - (name: "Performance Gauge") - (version: "0.1.8") - (author: "Eugene Retunsky") - (about: "A tool for gauging performance of network services") - (@arg CONCURRENCY: --concurrency -c +takes_value "Concurrent clients. Default `1`.") - (@group duration => - (@arg NUMBER_OF_REQUESTS: --num_req -n +takes_value "Number of requests per client.") - (@arg DURATION: --duration -d +takes_value "Duration of the test.") - ) - (@arg TEST_CASE_NAME: --name -N +takes_value "Test case name. Optional. Can be used for tagging metrics.") - (@arg RATE: --rate -r +takes_value "Request rate per second. E.g. 100 or 0.1. By default no limit.") - (@arg RATE_STEP: --rate_step +takes_value "Rate increase step (until it reaches --rate_max).") - (@arg RATE_MAX: --rate_max +takes_value "Max rate per second. Requires --rate-step") - (@arg MAX_RATE_ITERATIONS: --max_iter -m +takes_value "The number of iterations with the max rate. By default `1`.") - (@arg CONTINUOUS: --continuous "If it's a part of a continuous run. In this case metrics are not reset at the end to avoid saw-like plots.") - (@arg PROMETHEUS_ADDR: --prometheus +takes_value "If you'd like to send metrics to Prometheus PushGateway, specify the server URL. E.g. 10.0.0.1:9091") - (@arg PROMETHEUS_JOB: --prometheus_job +takes_value "Prometheus Job (by default `pushgateway`)") - (@subcommand http => - (about: "Run in HTTP(S) mode") - (version: "0.1.8") - (@arg IGNORE_CERT: --ignore_cert "Allow self signed certificates.") - (@arg CONN_REUSE: --conn_reuse "If connections should be re-used") - (@arg HTTP2_ONLY: --http2_only "Enforce HTTP/2 only") - (@arg TARGET: +required ... "Target, e.g. https://my-service.com:8443/8kb Can be multiple ones (with random choice balancing)") - (@arg METHOD: --method -M +takes_value "Method. By default GET") - (@arg HEADER: --header -H ... "Headers in \"Name:Value\" form. Can be provided multiple times.") - (@arg STOP_ON_ERRORS: --error_stop -E ... "Stop immediately on error codes. E.g. `-E 401 -E 403`") - (@arg BODY: --body -B +takes_value "Body of the request. Could be either `random://[0-9]+`, `file://$filename` or `base64://${valid_base64}`. Optional.") - ) - ).get_matches(); + let cli = Cli::parse(); - let test_case_name = matches.value_of("TEST_CASE_NAME").map(|s| s.to_string()); - let concurrency = matches.value_of("CONCURRENCY").unwrap_or("1"); - let rate_per_second = matches.value_of("RATE"); - let rate_step = matches.value_of("RATE_STEP"); - let rate_max = matches.value_of("RATE_MAX"); - let max_rate_iterations = matches.value_of("MAX_RATE_ITERATIONS").unwrap_or("1"); + let concurrency = cli.concurrency; + let rate_per_second = cli.rate; + let rate_step = cli.rate_step; + let rate_max = cli.rate_max; + let max_rate_iterations = cli.max_iter; - let duration = matches.value_of("DURATION").map(|d| { - humantime::Duration::from_str(d) + let duration = cli.duration.as_ref().map(|d| { + humantime::Duration::from_str(d.as_str()) .expect("Illegal duration") .into() }); - let number_of_requests = matches - .value_of("NUMBER_OF_REQUESTS") - .map(|n| parse_num(n, "Illegal number for NUMBER_OF_REQUESTS")); + let number_of_requests = cli.num_req; if duration.is_none() && number_of_requests.is_none() { panic!("Either the number of requests or the test duration must be specified"); @@ -102,43 +153,37 @@ impl BenchmarkConfig { rate_per_second.expect("RATE is required if RATE_MAX is specified"); let rate_step = rate_step.expect("RATE_STEP is required if RATE_MAX is specified"); RateLadderBuilder::default() - .start(parse_num(rate_per_second, "Cannot parse RATE")) - .end(parse_num(rate_max, "Cannot parse RATE_MAX")) - .rate_increment(Some(parse_num(rate_step, "Cannot parse RATE_STEP"))) + .start(rate_per_second) + .end(rate_max) + .rate_increment(Some(rate_step)) .step_duration(duration) .step_requests(number_of_requests) - .max_rate_iterations(parse_num( - max_rate_iterations, - "Cannot parse MAX_RATE_ITERATIONS", - )) + .max_rate_iterations(max_rate_iterations) .build() .expect("RateLadderBuilder failed") } else { - let rps = parse_num(rate_per_second.unwrap_or("0"), "Cannot parse RATE"); + let rps = rate_per_second.unwrap_or(0.0); RateLadderBuilder::default() .start(rps) .end(rps) .rate_increment(None) .step_duration(duration) .step_requests(number_of_requests) - .max_rate_iterations(parse_num( - max_rate_iterations, - "Cannot parse MAX_RATE_ITERATIONS", - )) + .max_rate_iterations(max_rate_iterations) .build() .expect("RateLadderBuilder failed") }; Ok(BenchmarkConfigBuilder::default() - .name(test_case_name.clone()) + .name(cli.name.clone()) .rate_ladder(rate_ladder) - .concurrency(parse_num(concurrency, "Cannot parse CONCURRENCY")) + .concurrency(concurrency) .verbose(false) - .continuous(matches.is_present("CONTINUOUS")) - .mode(BenchmarkConfig::build_mode(&matches)) + .continuous(cli.continuous) + .mode(BenchmarkConfig::build_mode(&cli)) .reporters(BenchmarkConfig::build_metric_destinations( - test_case_name, - matches, + cli.name.clone(), + &cli, )) .build() .expect("BenchmarkConfig failed")) @@ -160,7 +205,7 @@ impl BenchmarkConfig { #[cfg(feature = "report-to-prometheus")] fn build_metric_destinations( test_case_name: Option, - matches: ArgMatches, + args: &Cli, ) -> Vec> { use crate::prometheus_reporter::PrometheusReporter; use std::net::SocketAddr; @@ -171,69 +216,82 @@ impl BenchmarkConfig { test_case_name.clone(), ))]; - if let Some(prometheus_addr) = matches.value_of("PROMETHEUS_ADDR") { - if SocketAddr::from_str(prometheus_addr).is_err() { + if let Some(prometheus_addr) = &args.prometheus { + if SocketAddr::from_str(prometheus_addr.as_str()).is_err() { panic!("Illegal Prometheus Gateway addr `{}`", prometheus_addr); } metrics_destinations.push(Arc::new(PrometheusReporter::new( test_case_name, prometheus_addr.to_string(), - matches.value_of("PROMETHEUS_JOB"), + Some( + args.prometheus_job + .as_ref() + .unwrap_or(&"pushgateway".to_string()) + .clone() + .as_str(), + ), ))); } metrics_destinations } - fn build_mode(matches: &ArgMatches) -> BenchmarkMode { - let mode = if let Some(config) = matches.subcommand_matches("http") { - #[cfg(feature = "tls-boring")] - if config.is_present("IGNORE_CERT") { - println!("--ignore_cert is not supported for BoringSSL"); - exit(-1); - } + fn build_mode(args: &Cli) -> BenchmarkMode { + match &args.command { + Commands::Http(config) => { + #[cfg(feature = "tls-boring")] + if config.ignore_cert { + println!("--ignore_cert is not supported for BoringSSL"); + exit(-1); + } - let http_config = HttpBenchAdapterBuilder::default() - .config( - HttpClientConfigBuilder::default() - .ignore_cert(config.is_present("IGNORE_CERT")) - .conn_reuse(config.is_present("CONN_REUSE")) - .http2_only(config.is_present("HTTP2_ONLY")) - .stop_on_errors(BenchmarkConfig::parse_list(config, "STOP_ON_ERRORS")) - .build() - .expect("HttpClientConfigBuilder failed"), - ) - .request( - HttpRequestBuilder::default() - .url( - config - .values_of("TARGET") - .expect("misconfiguration for TARGET") - .map(|s| s.to_string()) - .collect(), - ) - .method(config.value_of("METHOD").unwrap_or("GET").to_string()) - .headers(BenchmarkConfig::parse_multiple_values(config, "HEADER")) - .body(BenchmarkConfig::generate_body(config)) - .build() - .expect("HttpRequestBuilder failed"), - ) - .build() - .expect("BenchmarkModeBuilder failed"); - BenchmarkMode::Http(http_config) - } else { - println!("Run `perf-gauge help` to see program options."); - exit(1); - }; - mode + let http_config = HttpBenchAdapterBuilder::default() + .config( + HttpClientConfigBuilder::default() + .ignore_cert(config.ignore_cert) + .conn_reuse(config.conn_reuse) + .http2_only(config.http2_only) + .stop_on_errors(config.error_stop.clone()) + .build() + .expect("HttpClientConfigBuilder failed"), + ) + .request( + HttpRequestBuilder::default() + .url(config.target.clone()) + .method(config.method.as_ref().unwrap_or(&"GET".to_string()).clone()) + .headers( + config + .header + .iter() + .map(|s| { + let mut split = s.split(':'); + ( + split + .next() + .expect("Header name is missing") + .to_string(), + split.collect::>().join(":"), + ) + }) + .collect(), + ) + .body(BenchmarkConfig::generate_body(config)) + .build() + .expect("HttpRequestBuilder failed"), + ) + .build() + .expect("BenchmarkModeBuilder failed"); + BenchmarkMode::Http(http_config) + } + } } - fn generate_body(config: &ArgMatches) -> Vec { + fn generate_body(args: &HttpOptions) -> Vec { const RANDOM_PREFIX: &str = "random://"; const BASE64_PREFIX: &str = "base64://"; const FILE_PREFIX: &str = "file://"; - if let Some(body_value) = config.value_of("BODY") { + if let Some(body_value) = &args.body { if let Some(body_size) = body_value.strip_prefix(RANDOM_PREFIX) { BenchmarkConfig::generate_random_vec(body_size) } else if let Some(base64) = body_value.strip_prefix(BASE64_PREFIX) { @@ -268,33 +326,6 @@ impl BenchmarkConfig { buffer } - fn parse_multiple_values(config: &ArgMatches, id: &str) -> Vec<(String, String)> { - config - .values_of(id) - .map(|v| { - v.map(|s| { - let mut split = s.split(':'); - ( - split.next().expect("Header name is missing").to_string(), - split.collect::>().join(":"), - ) - }) - .collect() - }) - .unwrap_or_else(Vec::new) - } - - fn parse_list(config: &ArgMatches, id: &str) -> Vec { - if let Some(value) = config.value_of(id) { - value - .split(',') - .map(|s| parse_num::(s, "Cannot parse error code")) - .collect() - } else { - vec![] - } - } - pub fn new_bench_session(&mut self) -> BenchSession { BenchSessionBuilder::default() .concurrency(self.concurrency) @@ -314,12 +345,3 @@ impl fmt::Display for BenchmarkConfig { ) } } - -pub fn parse_num(s: &str, error_msg: &str) -> F { - s.parse() - .map_err(|_| { - println!("{}. Non-number value: {}", error_msg, s); - panic!("Cannot start"); - }) - .unwrap() -} From 1514505a3e05063b7e7d66f1dd5f0d2aa0a1b312 Mon Sep 17 00:00:00 2001 From: xnuter Date: Sat, 12 Mar 2022 20:39:42 -0800 Subject: [PATCH 11/12] Upgrade `clap` --- src/configuration.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/configuration.rs b/src/configuration.rs index 4111e08..751d558 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -18,8 +18,6 @@ use rand::Rng; use std::fs; use std::fs::File; use std::io::Read; -#[cfg(feature = "tls-boring")] -use std::process::exit; use std::str::FromStr; use std::sync::Arc; use tokio::io; @@ -192,9 +190,11 @@ impl BenchmarkConfig { #[cfg(not(feature = "report-to-prometheus"))] fn build_metric_destinations( test_case_name: Option, - matches: ArgMatches, + args: &Cli, ) -> Vec> { - if matches.value_of("PROMETHEUS_ADDR").is_some() { + use std::process::exit; + + if args.prometheus.is_some() { println!("Prometheus is not supported in this configuration"); exit(-1); } @@ -241,6 +241,8 @@ impl BenchmarkConfig { Commands::Http(config) => { #[cfg(feature = "tls-boring")] if config.ignore_cert { + use std::process::exit; + println!("--ignore_cert is not supported for BoringSSL"); exit(-1); } From c0177d61953649466238861462e08ac6d22d1390 Mon Sep 17 00:00:00 2001 From: xnuter Date: Sun, 13 Mar 2022 12:15:36 -0700 Subject: [PATCH 12/12] Additional tests --- src/bench_run.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/src/bench_run.rs b/src/bench_run.rs index 9bf5411..e4c48c9 100644 --- a/src/bench_run.rs +++ b/src/bench_run.rs @@ -92,12 +92,16 @@ impl BenchRun { ) -> Result<(), String> { let client = bench_protocol_adapter.build_client()?; - while self.has_more_work() && !STOP_ON_FATAL.load(Ordering::Relaxed) { + while self.has_more_work() { self.rate_limiter .acquire_one() .await .expect("Unexpected LeakyBucket.acquire error"); + if STOP_ON_FATAL.load(Ordering::Relaxed) { + break; + } + let request_stats = bench_protocol_adapter.send_request(&client).await; let fatal_error = request_stats.fatal_error; @@ -120,6 +124,7 @@ impl BenchRun { #[cfg(test)] mod tests { + use crate::bench_run::STOP_ON_FATAL; use crate::bench_session::RateLadderBuilder; use crate::configuration::BenchmarkMode::Http; use crate::configuration::{BenchmarkConfig, BenchmarkConfigBuilder}; @@ -128,6 +133,7 @@ mod tests { }; use crate::metrics::BenchRunMetrics; use mockito::mock; + use std::sync::atomic::Ordering; use std::time::Instant; #[tokio::test] @@ -152,7 +158,12 @@ mod tests { .build() .unwrap(), ) - .config(HttpClientConfigBuilder::default().build().unwrap()) + .config( + HttpClientConfigBuilder::default() + .stop_on_errors(vec![401]) + .build() + .unwrap(), + ) .build() .unwrap(); @@ -177,12 +188,15 @@ mod tests { let bench_run_stats = BenchRunMetrics::new(); + STOP_ON_FATAL.store(false, Ordering::Relaxed); + let bench_result = session .next() .expect("Must have runs") .run(bench_run_stats) .await; + assert!(!STOP_ON_FATAL.load(Ordering::Relaxed)); assert!(bench_result.is_ok()); let elapsed = Instant::now().duration_since(start).as_secs_f64(); @@ -202,4 +216,67 @@ mod tests { Some(&(request_count as i32)) ); } + + #[tokio::test] + async fn test_send_load_fatal_code() { + let body = "world"; + + let request_count = 100; + + let _m = mock("GET", "/1") + .with_status(401) + .with_header("content-type", "text/plain") + .with_body(body) + .expect(request_count) + .create(); + + let url = mockito::server_url().to_string(); + println!("Url: {}", url); + let http_adapter = HttpBenchAdapterBuilder::default() + .request( + HttpRequestBuilder::default() + .url(vec![format!("{}/1", url)]) + .build() + .unwrap(), + ) + .config( + HttpClientConfigBuilder::default() + .stop_on_errors(vec![401]) + .build() + .unwrap(), + ) + .build() + .unwrap(); + + let benchmark_config: BenchmarkConfig = BenchmarkConfigBuilder::default() + .rate_ladder( + RateLadderBuilder::default() + .start(request_count as f64) + .end(request_count as f64) + .rate_increment(None) + .step_duration(None) + .step_requests(Some(request_count)) + .build() + .expect("RateLadderBuilder failed"), + ) + .mode(Http(http_adapter.clone())) + .build() + .expect("BenchmarkConfig failed"); + + let mut session = benchmark_config.clone().new_bench_session(); + + let bench_run_stats = BenchRunMetrics::new(); + + STOP_ON_FATAL.store(false, Ordering::Relaxed); + + let bench_result = session + .next() + .expect("Must have runs") + .run(bench_run_stats) + .await; + + // must stop on fatal + assert!(STOP_ON_FATAL.load(Ordering::Relaxed)); + assert!(bench_result.is_ok()); + } }