add histogram to more core s3 calls

quickwit/quickwit-storage/src/metrics.rs

@@ -19,7 +19,8 @@
 
 // See https://prometheus.io/docs/practices/naming/
 
-use once_cell::sync::Lazy;
+use std::sync::LazyLock;
+
 use quickwit_common::metrics::{
     new_counter, new_counter_vec, new_gauge, new_histogram_vec, Histogram, IntCounter,
     IntCounterVec, IntGauge,
@@ -44,8 +45,14 @@ pub struct StorageMetrics {
 
     pub object_storage_delete_requests_total: IntCounter,
     pub object_storage_bulk_delete_requests_total: IntCounter,
-    pub object_storage_delete_request_duration: Histogram,
-    pub object_storage_bulk_delete_request_duration: Histogram,
+    pub object_storage_delete_request_duration:
+        LazyLock<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_bulk_delete_request_duration:
+        LazyLock<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_get_request_duration: LazyLock<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_put_request_duration: LazyLock<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_put_part_request_duration:
+        LazyLock<Histogram, Box<dyn Fn() -> Histogram + Send>>,
 }
 
 impl Default for StorageMetrics {
@@ -86,10 +93,26 @@ impl Default for StorageMetrics {
             ["action"],
             vec![0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0],
         );
-        let object_storage_delete_request_duration =
-            object_storage_request_duration.with_label_values(["delete_object"]);
-        let object_storage_bulk_delete_request_duration =
-            object_storage_request_duration.with_label_values(["delete_objects"]);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_delete_request_duration = LazyLock::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["delete_object"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_bulk_delete_request_duration = LazyLock::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["delete_objects"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_get_request_duration = LazyLock::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["get"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_put_request_duration = LazyLock::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["put"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_put_part_request_duration = LazyLock::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["put_part"])
+        }) as _);
 
         StorageMetrics {
             fast_field_cache: CacheMetrics::for_component("fastfields"),
@@ -142,6 +165,9 @@ impl Default for StorageMetrics {
             object_storage_bulk_delete_requests_total,
             object_storage_delete_request_duration,
             object_storage_bulk_delete_request_duration,
+            object_storage_get_request_duration,
+            object_storage_put_request_duration,
+            object_storage_put_part_request_duration,
         }
     }
 }
@@ -212,8 +238,8 @@ impl CacheMetrics {
 
 /// Storage counters exposes a bunch a set of storage/cache related metrics through a prometheus
 /// endpoint.
-pub static STORAGE_METRICS: Lazy<StorageMetrics> = Lazy::new(StorageMetrics::default);
+pub static STORAGE_METRICS: LazyLock<StorageMetrics> = LazyLock::new(StorageMetrics::default);
 
 #[cfg(test)]
-pub static CACHE_METRICS_FOR_TESTS: Lazy<CacheMetrics> =
-    Lazy::new(|| CacheMetrics::for_component("fortest"));
+pub static CACHE_METRICS_FOR_TESTS: LazyLock<CacheMetrics> =
+    LazyLock::new(|| CacheMetrics::for_component("fortest"));

quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs

@@ -294,6 +294,9 @@ impl S3CompatibleObjectStorage {
         crate::STORAGE_METRICS
             .object_storage_upload_num_bytes
             .inc_by(len);
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_put_request_duration
+            .start_timer();
 
         self.s3_client
             .put_object()
@@ -429,6 +432,9 @@ impl S3CompatibleObjectStorage {
         crate::STORAGE_METRICS
             .object_storage_upload_num_bytes
             .inc_by(part.len());
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_put_part_request_duration
+            .start_timer();
 
         let upload_part_output = self
             .s3_client
@@ -549,6 +555,9 @@ impl S3CompatibleObjectStorage {
         let range_str = range_opt.map(|range| format!("bytes={}-{}", range.start, range.end - 1));
 
         crate::STORAGE_METRICS.object_storage_get_total.inc();
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_get_request_duration
+            .start_timer();
 
         let get_object_output = self
             .s3_client