adapter: Add metric for /metrics payload size

We saw that the query log/query log ad hoc can cause the /metrics payload to grow, so this commit adds an easy way to determine the current payload size of that endpoint. It also creates an extensible struct that can be re-used for other http endpoints that may have interesting metrics to report. Change-Id: Ia5657ebbb2f471a3d1edf74b087b73bf081508b2 Reviewed-on: https://gerrit.readyset.name/c/readyset/+/6219 Tested-by: Buildkite CI Reviewed-by: Jason Brown <jason.b@readyset.io>
readysettech · Oct 17, 2023 · e2aac68 · e2aac68
1 parent d34ba16
commit e2aac68
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 1 deletion.
diff --git a/readyset-adapter/src/http_router.rs b/readyset-adapter/src/http_router.rs
@@ -11,6 +11,7 @@ use health_reporter::{HealthReporter as AdapterHealthReporter, State};
 use hyper::header::CONTENT_TYPE;
 use hyper::service::make_service_fn;
 use hyper::{self, Body, Method, Request, Response};
+use metrics::Gauge;
 use metrics_exporter_prometheus::PrometheusHandle;
 use readyset_alloc::{dump_stats, memory_and_per_thread_stats};
 use readyset_client_metrics::recorded;
@@ -38,6 +39,9 @@ pub struct NoriaAdapterHttpRouter {
     /// Used to retrieve the prometheus scrape's render as a String when servicing
     /// HTTP requests on /metrics.
     pub prometheus_handle: Option<PrometheusHandle>,
+
+    /// Used to record metrics related to http request handling.
+    pub metrics: HttpRouterMetrics,
 }
 
 impl NoriaAdapterHttpRouter {
@@ -204,7 +208,10 @@ impl Service<Request<Body>> for NoriaAdapterHttpRouter {
                 let body = self.prometheus_handle.as_ref().map(|x| x.render());
                 let res = res.header(CONTENT_TYPE, "text/plain");
                 let res = match body {
-                    Some(metrics) => res.body(hyper::Body::from(metrics)),
+                    Some(metrics) => {
+                        self.metrics.rec_metrics_payload_size(metrics.len());
+                        res.body(hyper::Body::from(metrics))
+                    }
                     None => res
                         .status(404)
                         .body(hyper::Body::from("Prometheus metrics were not enabled. To fix this, run the adapter with --prometheus-metrics".to_string())),
@@ -253,3 +260,24 @@ impl Service<Request<Body>> for NoriaAdapterHttpRouter {
         }
     }
 }
+
+#[derive(Clone)]
+pub struct HttpRouterMetrics {
+    /// The last seen size of the /metrics endpoint payload, in bytes
+    metrics_payload_size: Gauge,
+}
+
+impl Default for HttpRouterMetrics {
+    fn default() -> Self {
+        Self {
+            metrics_payload_size: metrics::register_gauge!(recorded::METRICS_PAYLOAD_SIZE_BYTES),
+        }
+    }
+}
+
+impl HttpRouterMetrics {
+    /// Record the size of the /metrics payload in bytes
+    pub(super) fn rec_metrics_payload_size(&self, payload_size: usize) {
+        self.metrics_payload_size.set(payload_size as f64);
+    }
+}
diff --git a/readyset-client-metrics/src/recorded.rs b/readyset-client-metrics/src/recorded.rs
@@ -79,3 +79,6 @@ pub const QUERY_LOG_VIEW_NOT_FOUND: &str = "readyset_query_log_view_not_found";
 
 /// Counter: The number of errors due to RPC failures.
 pub const QUERY_LOG_RPC_ERRORS: &str = "readyset_query_log_rpc_errors";
+
+/// Gauge: The last seen size in bytes of a /metrics payload.
+pub const METRICS_PAYLOAD_SIZE_BYTES: &str = "readyset_metrics_payload_size_bytes";
diff --git a/readyset/src/lib.rs b/readyset/src/lib.rs
@@ -830,6 +830,7 @@ where
             prometheus_handle: prometheus_handle.clone(),
             health_reporter: health_reporter.clone(),
             failpoint_channel: tx,
+            metrics: Default::default(),
         };
 
         let router_shutdown_rx = shutdown_rx.clone();