chore: measure the state height difference during query scheduling

rs/execution_environment/src/lib.rs

@@ -172,6 +172,7 @@ impl ExecutionServices {
             Arc::clone(&sync_query_handler) as Arc<_>,
             query_scheduler.clone(),
             Arc::clone(&state_reader),
+            metrics_registry,
         );
         let ingress_filter = IngressFilterServiceImpl::new_service(
             query_scheduler.clone(),

rs/execution_environment/src/query_handler.rs

@@ -39,6 +39,7 @@ use ic_types::{
     },
     CanisterId, NumInstructions, PrincipalId,
 };
+use prometheus::Histogram;
 use serde::Serialize;
 use std::convert::Infallible;
 use std::str::FromStr;
@@ -113,12 +114,30 @@ pub struct InternalHttpQueryHandler {
     query_cache: query_cache::QueryCache,
 }
 
+#[derive(Clone)]
+struct HttpQueryHandlerMetrics {
+    pub height_diff_during_query_scheduling: Histogram,
+}
+
+impl HttpQueryHandlerMetrics {
+    pub fn new(metrics_registry: &MetricsRegistry) -> Self {
+        Self {
+            height_diff_during_query_scheduling: metrics_registry.histogram(
+                "execution_query_height_diff_during_query_scheduling",
+                "The height difference between the latest certified height before query scheduling and state height used for execution",
+                vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 20.0, 50.0, 100.0],
+            ),
+        }
+    }
+}
+
 #[derive(Clone)]
 /// Struct that is responsible for handling queries sent by user.
 pub(crate) struct HttpQueryHandler {
     internal: Arc<dyn QueryHandler<State = ReplicatedState>>,
     state_reader: Arc<dyn StateReader<State = ReplicatedState>>,
     query_scheduler: QueryScheduler,
+    metrics: Arc<HttpQueryHandlerMetrics>,
 }
 
 impl InternalHttpQueryHandler {
@@ -351,11 +370,13 @@ impl HttpQueryHandler {
         internal: Arc<dyn QueryHandler<State = ReplicatedState>>,
         query_scheduler: QueryScheduler,
         state_reader: Arc<dyn StateReader<State = ReplicatedState>>,
+        metrics_registry: &MetricsRegistry,
     ) -> QueryExecutionService {
         BoxCloneService::new(Self {
             internal,
             state_reader,
             query_scheduler,
+            metrics: Arc::new(HttpQueryHandlerMetrics::new(metrics_registry)),
         })
     }
 }
@@ -391,6 +412,8 @@ impl Service<(UserQuery, Option<CertificateDelegation>)> for HttpQueryHandler {
         let state_reader = Arc::clone(&self.state_reader);
         let (tx, rx) = oneshot::channel();
         let canister_id = query.receiver;
+        let latest_certified_height_pre_schedule = state_reader.latest_certified_height();
+        let http_query_handler_metrics = Arc::clone(&self.metrics);
         self.query_scheduler.push(canister_id, move || {
             let start = std::time::Instant::now();
             if !tx.is_closed() {
@@ -408,6 +431,15 @@ impl Service<(UserQuery, Option<CertificateDelegation>)> for HttpQueryHandler {
                 ) {
                     Some((state, cert)) => {
                         let time = state.get_ref().metadata.batch_time;
+
+                        let certified_height_used_for_execution = state.height();
+                        let height_diff = certified_height_used_for_execution
+                            .get()
+                            .saturating_sub(latest_certified_height_pre_schedule.get());
+                        http_query_handler_metrics
+                            .height_diff_during_query_scheduling
+                            .observe(height_diff as f64);
+
                         let result = internal.query(query, state, cert);
 
                         let response = match result {