Merge b894c71 into 4d9b724

xnuter · Jul 21, 2021 · 70207b4 · 70207b4
2 parents 4d9b724 + b894c71
commit 70207b4
Show file tree

Hide file tree

Showing 4 changed files with 331 additions and 56 deletions.
diff --git a/src/bench_run.rs b/src/bench_run.rs
@@ -178,8 +178,11 @@ mod tests {
 
         let stats = bench_result.unwrap();
 
-        assert_eq!(body.len() * request_count, stats.total_bytes);
-        assert_eq!(request_count, stats.total_requests);
-        assert_eq!(stats.summary.get("200 OK"), Some(&(request_count as i32)));
+        assert_eq!(body.len() * request_count, stats.combined.total_bytes);
+        assert_eq!(request_count, stats.combined.total_requests);
+        assert_eq!(
+            stats.combined.summary.get("200 OK"),
+            Some(&(request_count as i32))
+        );
     }
 }
diff --git a/src/configuration.rs b/src/configuration.rs
@@ -169,7 +169,7 @@ impl BenchmarkConfig {
                 panic!("Illegal Prometheus Gateway addr `{}`", prometheus_addr);
             }
             metrics_destinations.push(Arc::new(PrometheusReporter::new(
-                test_case_name.clone(),
+                test_case_name,
                 prometheus_addr.to_string(),
                 matches.value_of("PROMETHEUS_JOB"),
             )));

diff --git a/src/metrics.rs b/src/metrics.rs
@@ -18,6 +18,12 @@ pub struct DefaultConsoleReporter {
 
 #[derive(Clone)]
 pub struct BenchRunMetrics {
+    pub(crate) combined: BenchRunMetricsItem,
+    pub(crate) by_operation: HashMap<String, BenchRunMetricsItem>,
+}
+
+#[derive(Clone)]
+pub struct BenchRunMetricsItem {
     pub(crate) bench_begin: Instant,
     pub(crate) total_bytes: usize,
     pub(crate) total_requests: usize,
@@ -27,9 +33,15 @@ pub struct BenchRunMetrics {
     pub(crate) error_latency: Histogram,
 }
 
-/// Default reporter that prints stats to console.
 #[derive(Serialize)]
 struct BenchRunReport {
+    combined: BenchRunReportItem,
+    by_operation: HashMap<String, BenchRunReportItem>,
+}
+
+/// Default reporter that prints stats to console.
+#[derive(Serialize)]
+struct BenchRunReportItem {
     test_case_name: Option<String>,
     duration: Duration,
     total_bytes: usize,
@@ -47,9 +59,30 @@ pub struct RequestStats {
     pub bytes_processed: usize,
     pub status: String,
     pub duration: Duration,
+    #[builder(default = "None")]
+    pub operation_name: Option<String>,
 }
 
 impl BenchRunMetrics {
+    pub fn new() -> Self {
+        Self {
+            combined: BenchRunMetricsItem::new(),
+            by_operation: HashMap::new(),
+        }
+    }
+
+    pub fn report_request(&mut self, stats: RequestStats) {
+        self.combined.report_request(&stats);
+        if let Some(operation_name) = stats.operation_name.as_ref() {
+            self.by_operation
+                .entry(operation_name.to_owned())
+                .or_insert_with(BenchRunMetricsItem::new)
+                .report_request(&stats);
+        }
+    }
+}
+
+impl BenchRunMetricsItem {
     pub fn new() -> Self {
         Self {
             bench_begin: Instant::now(),
@@ -62,7 +95,7 @@ impl BenchRunMetrics {
         }
     }
 
-    pub fn report_request(&mut self, stats: RequestStats) {
+    pub fn report_request(&mut self, stats: &RequestStats) {
         self.total_requests += 1;
         if stats.is_success {
             self.successful_requests += 1;
@@ -75,7 +108,10 @@ impl BenchRunMetrics {
                 .unwrap_or_default();
         }
         self.total_bytes += stats.bytes_processed;
-        self.summary.entry(stats.status).or_insert(0).add_assign(1);
+        self.summary
+            .entry(stats.status.to_owned())
+            .or_insert(0)
+            .add_assign(1);
     }
 
     pub fn truncated_mean(histogram: &Histogram, threshold: f64) -> u64 {
@@ -110,8 +146,8 @@ impl BenchRunMetrics {
     }
 }
 
-impl BenchRunReport {
-    fn summary_ordered(metrics: &BenchRunMetrics) -> Vec<(String, i32)> {
+impl BenchRunReportItem {
+    fn summary_ordered(metrics: &BenchRunMetricsItem) -> Vec<(String, i32)> {
         let mut pairs: Vec<(String, i32)> = metrics
             .summary
             .iter()
@@ -130,7 +166,7 @@ impl BenchRunReport {
         pairs
     }
 
-    fn latency_summary(metrics: &BenchRunMetrics) -> Vec<(String, u64)> {
+    fn latency_summary(metrics: &BenchRunMetricsItem) -> Vec<(String, u64)> {
         // for simplicity of reporting we merge both latency
         // into a single histogram.
         let mut latency = metrics.success_latency.clone();
@@ -163,21 +199,27 @@ impl BenchRunReport {
             ("StdDev".to_string(), latency.stddev().unwrap_or_default()),
             (
                 "tm95".to_string(),
-                BenchRunMetrics::truncated_mean(&latency, 5.0),
+                BenchRunMetricsItem::truncated_mean(&latency, 5.0),
             ),
             (
                 "tm99".to_string(),
-                BenchRunMetrics::truncated_mean(&latency, 1.0),
+                BenchRunMetricsItem::truncated_mean(&latency, 1.0),
             ),
             (
                 "tm99.9".to_string(),
-                BenchRunMetrics::truncated_mean(&latency, 0.1),
+                BenchRunMetricsItem::truncated_mean(&latency, 0.1),
             ),
         ]
     }
 }
 
 impl fmt::Display for BenchRunReport {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "{}", self.combined)
+    }
+}
+
+impl fmt::Display for BenchRunReportItem {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let name = match self.test_case_name.as_ref() {
             None => String::new(),
@@ -273,12 +315,38 @@ impl DefaultConsoleReporter {
         Self { test_case_name }
     }
 
+    fn sorted_operations(metrics: &BenchRunMetrics) -> Vec<String> {
+        let sorted_operation_name: Vec<String> =
+            metrics.by_operation.keys().map(|s| s.to_owned()).collect();
+        sorted_operation_name
+    }
+
     fn build_report(&self, metrics: &BenchRunMetrics) -> BenchRunReport {
+        let mut by_operation = HashMap::new();
+        let sorted_operation_name = DefaultConsoleReporter::sorted_operations(metrics);
+        for operation in sorted_operation_name {
+            by_operation.insert(
+                operation.to_owned(),
+                self.build_item_report(
+                    &metrics
+                        .by_operation
+                        .get(&operation)
+                        .expect("Operation key cannot be missing"),
+                ),
+            );
+        }
+        BenchRunReport {
+            combined: self.build_item_report(&metrics.combined),
+            by_operation,
+        }
+    }
+
+    fn build_item_report(&self, metrics: &BenchRunMetricsItem) -> BenchRunReportItem {
         let successful_requests = metrics.successful_requests as usize;
         let total_requests = metrics.total_requests as usize;
         let total_bytes = metrics.total_bytes as usize;
         let duration = Instant::now().duration_since(metrics.bench_begin);
-        BenchRunReport {
+        BenchRunReportItem {
             test_case_name: self
                 .test_case_name
                 .as_ref()
@@ -290,8 +358,8 @@ impl DefaultConsoleReporter {
             success_rate: successful_requests as f64 * 100. / total_requests as f64,
             rate_per_second: total_requests as f64 / duration.as_secs_f64(),
             bitrate_mbps: total_bytes as f64 / duration.as_secs_f64() * 8. / 1_000_000.,
-            response_code_summary: BenchRunReport::summary_ordered(metrics),
-            latency_summary: BenchRunReport::latency_summary(metrics),
+            response_code_summary: BenchRunReportItem::summary_ordered(metrics),
+            latency_summary: BenchRunReportItem::latency_summary(metrics),
         }
     }
 }
@@ -323,11 +391,13 @@ mod tests {
                 bytes_processed: 0,
                 status: code,
                 duration: Default::default(),
+                operation_name: None,
             });
         }
 
         let mut ordered_summary = DefaultConsoleReporter::new(None)
             .build_report(&metrics)
+            .combined
             .response_code_summary
             .into_iter();
         assert_eq!(
@@ -350,11 +420,12 @@ mod tests {
                 bytes_processed: 0,
                 status: "200 OK".to_string(),
                 duration: Duration::from_micros(i),
+                operation_name: None,
             });
         }
 
         let report = DefaultConsoleReporter::new(None).build_report(&metrics);
-        let mut items = report.latency_summary.into_iter();
+        let mut items = report.combined.latency_summary.into_iter();
 
         assert_eq!(Some(("Min".to_string(), 0)), items.next());
         assert_eq!(Some(("p50".to_string(), 500)), items.next());
@@ -367,6 +438,73 @@ mod tests {
         assert_eq!(Some(("StdDev".to_string(), 289)), items.next());
     }
 
+    #[test]
+    fn test_by_operation_reporting() {
+        let mut metrics = BenchRunMetrics::new();
+        for i in 0..1000 {
+            metrics.report_request(RequestStats {
+                is_success: true,
+                bytes_processed: 0,
+                status: "200 OK".to_string(),
+                duration: Duration::from_micros(i),
+                operation_name: if i % 2 == 0 {
+                    Some("OperationA".to_string())
+                } else {
+                    Some("OperationB".to_string())
+                },
+            });
+        }
+
+        let report = DefaultConsoleReporter::new(None).build_report(&metrics);
+        let mut items = report.combined.latency_summary.to_owned().into_iter();
+
+        assert_eq!(Some(("Min".to_string(), 0)), items.next());
+        assert_eq!(Some(("p50".to_string(), 500)), items.next());
+        assert_eq!(Some(("p90".to_string(), 900)), items.next());
+        assert_eq!(Some(("p99".to_string(), 990)), items.next());
+        assert_eq!(Some(("p99.9".to_string(), 999)), items.next());
+        assert_eq!(Some(("p99.99".to_string(), 999)), items.next());
+        assert_eq!(Some(("Max".to_string(), 999)), items.next());
+        assert_eq!(Some(("Mean".to_string(), 500)), items.next());
+        assert_eq!(Some(("StdDev".to_string(), 289)), items.next());
+
+        assert_eq!(report.by_operation.len(), 2);
+
+        let mut items = report
+            .by_operation
+            .get("OperationA")
+            .unwrap()
+            .latency_summary
+            .to_owned()
+            .into_iter();
+        assert_eq!(Some(("Min".to_string(), 0)), items.next());
+        assert_eq!(Some(("p50".to_string(), 500)), items.next());
+        assert_eq!(Some(("p90".to_string(), 900)), items.next());
+        assert_eq!(Some(("p99".to_string(), 990)), items.next());
+        assert_eq!(Some(("p99.9".to_string(), 998)), items.next());
+        assert_eq!(Some(("p99.99".to_string(), 998)), items.next());
+        assert_eq!(Some(("Max".to_string(), 998)), items.next());
+        assert_eq!(Some(("Mean".to_string(), 499)), items.next());
+        assert_eq!(Some(("StdDev".to_string(), 289)), items.next());
+
+        let mut items = report
+            .by_operation
+            .get("OperationB")
+            .unwrap()
+            .latency_summary
+            .to_owned()
+            .into_iter();
+        assert_eq!(Some(("Min".to_string(), 1)), items.next());
+        assert_eq!(Some(("p50".to_string(), 501)), items.next());
+        assert_eq!(Some(("p90".to_string(), 901)), items.next());
+        assert_eq!(Some(("p99".to_string(), 991)), items.next());
+        assert_eq!(Some(("p99.9".to_string(), 999)), items.next());
+        assert_eq!(Some(("p99.99".to_string(), 999)), items.next());
+        assert_eq!(Some(("Max".to_string(), 999)), items.next());
+        assert_eq!(Some(("Mean".to_string(), 501)), items.next());
+        assert_eq!(Some(("StdDev".to_string(), 289)), items.next());
+    }
+
     #[test]
     fn test_has_more_work_request_limit() {
         let requests = 10;
@@ -399,6 +537,7 @@ mod tests {
                 bytes_processed: 0,
                 status: "200 OK".to_string(),
                 duration: Duration::from_micros(i),
+                operation_name: None,
             });
         }