add histogram to more core s3 calls

and make their histograms appear only when the endpoint has been called

Author: trinity-1686a

quickwit/quickwit-storage/src/metrics.rs

@@ -44,8 +44,13 @@ pub struct StorageMetrics {
 
     pub object_storage_delete_requests_total: IntCounter,
     pub object_storage_bulk_delete_requests_total: IntCounter,
-    pub object_storage_delete_request_duration: Histogram,
-    pub object_storage_bulk_delete_request_duration: Histogram,
+    pub object_storage_delete_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_bulk_delete_request_duration:
+        Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_get_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_put_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
+    pub object_storage_put_part_request_duration:
+        Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
 }
 
 impl Default for StorageMetrics {
@@ -86,10 +91,26 @@ impl Default for StorageMetrics {
             ["action"],
             vec![0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0],
         );
-        let object_storage_delete_request_duration =
-            object_storage_request_duration.with_label_values(["delete_object"]);
-        let object_storage_bulk_delete_request_duration =
-            object_storage_request_duration.with_label_values(["delete_objects"]);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_delete_request_duration = Lazy::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["delete_object"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_bulk_delete_request_duration = Lazy::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["delete_objects"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_get_request_duration = Lazy::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["get"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_put_request_duration = Lazy::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["put"])
+        }) as _);
+        let object_storage_request_duration_clone = object_storage_request_duration.clone();
+        let object_storage_put_part_request_duration = Lazy::new(Box::new(move || {
+            object_storage_request_duration_clone.with_label_values(["put_part"])
+        }) as _);
 
         StorageMetrics {
             fast_field_cache: CacheMetrics::for_component("fastfields"),
@@ -142,6 +163,9 @@ impl Default for StorageMetrics {
             object_storage_bulk_delete_requests_total,
             object_storage_delete_request_duration,
             object_storage_bulk_delete_request_duration,
+            object_storage_get_request_duration,
+            object_storage_put_request_duration,
+            object_storage_put_part_request_duration,
         }
     }
 }

quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs

@@ -294,6 +294,9 @@ impl S3CompatibleObjectStorage {
         crate::STORAGE_METRICS
             .object_storage_upload_num_bytes
             .inc_by(len);
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_put_request_duration
+            .start_timer();
 
         self.s3_client
             .put_object()
@@ -429,6 +432,9 @@ impl S3CompatibleObjectStorage {
         crate::STORAGE_METRICS
             .object_storage_upload_num_bytes
             .inc_by(part.len());
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_put_part_request_duration
+            .start_timer();
 
         let upload_part_output = self
             .s3_client
@@ -549,6 +555,9 @@ impl S3CompatibleObjectStorage {
         let range_str = range_opt.map(|range| format!("bytes={}-{}", range.start, range.end - 1));
 
         crate::STORAGE_METRICS.object_storage_get_total.inc();
+        let _timer = crate::STORAGE_METRICS
+            .object_storage_get_request_duration
+            .start_timer();
 
         let get_object_output = self
             .s3_client