From 677b3b75267f8b0103f646466b75b09903106dd4 Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 10:55:38 +0100 Subject: [PATCH 1/7] =?UTF-8?q?fix=20=F0=9F=90=9E:=20=20Fixed=20minor=20me?= =?UTF-8?q?trics=20problems?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gateway-state/src/lib.rs | 4 +- .../src/gateway_metrics.rs | 7 +- telemetry/dashboards/ic_metrics.json | 81 +++++++++++++------ 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs index 80077aea..6276d857 100644 --- a/src/gateway-state/src/lib.rs +++ b/src/gateway-state/src/lib.rs @@ -109,8 +109,8 @@ impl GatewayState { // therefore there is no need to do anything else here } - pub fn get_clients_count(&self, canister_id: CanisterPrincipal) -> usize { - if let Some(poller_state) = self.inner.data.get(&canister_id) { + pub fn get_clients_count(&self, canister_id: &CanisterPrincipal) -> usize { + if let Some(poller_state) = self.inner.data.get(canister_id) { poller_state.len() } else { 0 diff --git a/src/ic-websocket-gateway/src/gateway_metrics.rs b/src/ic-websocket-gateway/src/gateway_metrics.rs index f2a08b32..8be7d454 100644 --- a/src/ic-websocket-gateway/src/gateway_metrics.rs +++ b/src/ic-websocket-gateway/src/gateway_metrics.rs @@ -1,4 +1,4 @@ -use metrics::{describe_gauge, describe_histogram, gauge}; +use metrics::{describe_counter, describe_gauge, describe_histogram, gauge}; use metrics_exporter_prometheus::PrometheusBuilder; use metrics_util::MetricKindMask; use std::error::Error; @@ -9,12 +9,13 @@ use std::time::Duration; pub fn init_metrics(address: &str) -> Result<(), Box> { let builder = PrometheusBuilder::new().with_http_listener(SocketAddr::from_str(address)?); - // Set the idle timeout for counters and histograms to 30 seconds then the metrics are removed from the registry + // Set the idle timeout for counters and histograms to 10 seconds then the metrics are removed from the registry builder - .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(30))) + .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(10))) .install() .expect("failed to install Prometheus recorder"); + describe_counter!("client_connected_count", "Each time that a client connects it emits a point"); describe_gauge!( "clients_connected", "The number of clients currently connected" diff --git a/telemetry/dashboards/ic_metrics.json b/telemetry/dashboards/ic_metrics.json index 2eafcd6a..938d6444 100644 --- a/telemetry/dashboards/ic_metrics.json +++ b/telemetry/dashboards/ic_metrics.json @@ -352,7 +352,7 @@ "axisColorMode": "text", "axisLabel": "Number", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", @@ -405,7 +405,7 @@ "y": 18 }, "id": 6, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -427,11 +427,11 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(instance) (max_over_time(clients_connected[30s]))", + "expr": "sum by(instance) (max_over_time(client_connected_count[$__interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "interval": "1h", + "interval": "5m", "legendFormat": "__auto", "range": true, "refId": "A", @@ -458,7 +458,7 @@ "axisColorMode": "text", "axisLabel": "Number", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", @@ -511,7 +511,7 @@ "y": 18 }, "id": 7, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -533,11 +533,11 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(instance) (max_over_time(clients_connected[30s]))", + "expr": "sum by(instance) (max_over_time(client_connected_count[$__interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "interval": "1d", + "interval": "10m", "legendFormat": "__auto", "range": true, "refId": "A", @@ -616,7 +616,7 @@ "y": 26 }, "id": 2, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -720,7 +720,7 @@ "y": 26 }, "id": 8, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -776,7 +776,39 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "max": 1, @@ -804,20 +836,18 @@ "y": 35 }, "id": 12, + "interval": "10s", "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "10.4.0", "targets": [ @@ -839,7 +869,7 @@ } ], "title": "Gateway status", - "type": "gauge" + "type": "timeseries" }, { "datasource": { @@ -909,6 +939,7 @@ "y": 35 }, "id": 13, + "interval": "10s", "options": { "legend": { "calcs": [], @@ -957,6 +988,6 @@ "timezone": "browser", "title": "IC Metrics", "uid": "cdfh3d16g8dtsf", - "version": 9, + "version": 15, "weekStart": "" } From 9fb19a0c65f1d999006008077d920f3d3a549494 Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 11:53:30 +0100 Subject: [PATCH 2/7] =?UTF-8?q?refactor=20=E2=9A=99:=20=20Refactored=20met?= =?UTF-8?q?rics=20according=20to=20the=20meeting=20requested=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + src/gateway-state/Cargo.toml | 3 +- src/gateway-state/src/lib.rs | 35 +++++++++++++------ .../src/canister_poller.rs | 8 +++++ .../src/client_session_handler.rs | 19 ++-------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 311a7ee0..0192987a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,6 +917,7 @@ dependencies = [ "canister-utils", "dashmap", "ic-agent", + "metrics", "tokio", "tracing", ] diff --git a/src/gateway-state/Cargo.toml b/src/gateway-state/Cargo.toml index 72bec7a3..faf9ec8a 100644 --- a/src/gateway-state/Cargo.toml +++ b/src/gateway-state/Cargo.toml @@ -11,4 +11,5 @@ dashmap = "5.5.3" ic-agent = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -canister-utils = { version = "1.0.0", path = "../canister-utils" } \ No newline at end of file +metrics = "0.22.1" +canister-utils = { version = "1.0.0", path = "../canister-utils" } diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs index 6276d857..d3c6fae5 100644 --- a/src/gateway-state/src/lib.rs +++ b/src/gateway-state/src/lib.rs @@ -3,7 +3,8 @@ use dashmap::{mapref::entry::Entry, DashMap}; use ic_agent::export::Principal; use std::sync::Arc; use tokio::sync::mpsc::Sender; -use tracing::Span; +use tracing::{debug, Span}; +use metrics::{gauge}; /// State of the WS Gateway that can be shared between threads #[derive(Clone)] @@ -53,6 +54,11 @@ impl GatewayState { span: client_session_span, }, ); + + // Increment the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected.to_string()); + gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); // the poller shall not be started again None }, @@ -68,6 +74,11 @@ impl GatewayState { }, ); entry.insert(Arc::clone(&poller_state)); + + // Increment the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected.to_string()); + gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); // the poller shall be started Some(poller_state) }, @@ -98,6 +109,11 @@ impl GatewayState { // if this is encountered it might indicate a race condition unreachable!("Client key not found in poller state"); } + + // Decrement the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected.to_string()); + gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); // even if this is the last client session for the canister, do not remove the canister from the gateway state // this will be done by the poller task } @@ -109,14 +125,6 @@ impl GatewayState { // therefore there is no need to do anything else here } - pub fn get_clients_count(&self, canister_id: &CanisterPrincipal) -> usize { - if let Some(poller_state) = self.inner.data.get(canister_id) { - poller_state.len() - } else { - 0 - } - } - pub fn get_active_pollers_count(&self) -> usize { self.inner.data.len() } @@ -149,7 +157,14 @@ impl GatewayState { // returns 'ClientRemovalResult::Removed' if the client was removed, 'ClientRemovalResult::Vacant' if there was no such client return { match poller_state.remove(&client_key) { - Some(_) => ClientRemovalResult::Removed(client_key), + Some(_) => { + // Decrement the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected.to_string()); + gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); + + ClientRemovalResult::Removed(client_key) + }, None => ClientRemovalResult::Vacant, } }; diff --git a/src/ic-websocket-gateway/src/canister_poller.rs b/src/ic-websocket-gateway/src/canister_poller.rs index d095eba8..941f3474 100644 --- a/src/ic-websocket-gateway/src/canister_poller.rs +++ b/src/ic-websocket-gateway/src/canister_poller.rs @@ -8,6 +8,7 @@ use gateway_state::{ }; use ic_agent::{agent::RejectCode, Agent, AgentError}; use std::{sync::Arc, time::Duration}; +use metrics::gauge; use tokio::{sync::mpsc::Sender, time::timeout}; use tracing::{error, span, trace, warn, Instrument, Level, Span}; @@ -77,6 +78,11 @@ impl CanisterPoller { // this enables to crawl polling iterations in reverse chronological order polling_iteration_span.follows_from(previous_polling_iteration_span.id()); } + + // register the number of active clients + let clients_connected = self.poller_state.len(); + gauge!("clients_connected", "canister_id" => self.canister_id.to_string()).set(clients_connected as f64); + if let Err(e) = self .poll_and_relay() .instrument(polling_iteration_span.clone()) @@ -107,6 +113,7 @@ impl CanisterPoller { } pub async fn poll_and_relay(&mut self) -> Result<(), String> { + //TODO start the timer for the polling iteration let start_polling_instant = tokio::time::Instant::now(); match self.poll_canister().await? { @@ -144,6 +151,7 @@ impl CanisterPoller { PollingStatus::NoMessagesPolled => (), } + //TODO stop the timer for the polling iteration and log the duration // compute the amout of time to sleep for before polling again let effective_polling_interval = self.compute_effective_polling_interval(start_polling_instant); diff --git a/src/ic-websocket-gateway/src/client_session_handler.rs b/src/ic-websocket-gateway/src/client_session_handler.rs index e78b3701..a44212a9 100644 --- a/src/ic-websocket-gateway/src/client_session_handler.rs +++ b/src/ic-websocket-gateway/src/client_session_handler.rs @@ -9,7 +9,7 @@ use gateway_state::{CanisterPrincipal, ClientRemovalResult, GatewayState, Poller use ic_agent::Agent; use std::sync::Arc; use std::time::Instant; -use metrics::{gauge, histogram}; +use metrics::{counter, gauge, histogram}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc::{self, Receiver, Sender}, @@ -189,14 +189,9 @@ impl ClientSessionHandler { client_session_span.in_scope(|| { debug!("Client session opened"); - let canister_id = self.get_canister_id(&client_session); let client_key = self.get_client_key(&client_session); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); - + counter!("client_connected_count", "client_key" => client_key.to_string()).absolute(1); // Calculate the time it took to open the connection and record it using the timer started in ws_listener.rs let delta = self.start_connection_time.elapsed(); histogram!("connection_opening_time", "client_key" => client_key.to_string()).record(delta); @@ -215,11 +210,6 @@ impl ClientSessionHandler { .remove_client(canister_id, client_key.clone()); debug!("Client removed from gateway state"); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); - let delta = client_start_session_time.elapsed(); histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta); @@ -254,11 +244,6 @@ impl ClientSessionHandler { { debug!("Client removed from gateway state"); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); - let delta = client_start_session_time.elapsed(); histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta); From 4c666eb1033a1280c9ce4982e976a78f878cf331 Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 12:53:32 +0100 Subject: [PATCH 3/7] =?UTF-8?q?feat=20=F0=9F=A6=84:=20=20Implemented=20the?= =?UTF-8?q?=20polling=20interval=20metric=20and=20relative=20dashboards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/canister_poller.rs | 9 +- .../src/gateway_metrics.rs | 4 + telemetry/dashboards/ic_metrics.json | 230 +++++++++++++++++- 3 files changed, 228 insertions(+), 15 deletions(-) diff --git a/src/ic-websocket-gateway/src/canister_poller.rs b/src/ic-websocket-gateway/src/canister_poller.rs index 941f3474..2279b046 100644 --- a/src/ic-websocket-gateway/src/canister_poller.rs +++ b/src/ic-websocket-gateway/src/canister_poller.rs @@ -8,7 +8,7 @@ use gateway_state::{ }; use ic_agent::{agent::RejectCode, Agent, AgentError}; use std::{sync::Arc, time::Duration}; -use metrics::gauge; +use metrics::{gauge, histogram}; use tokio::{sync::mpsc::Sender, time::timeout}; use tracing::{error, span, trace, warn, Instrument, Level, Span}; @@ -113,7 +113,6 @@ impl CanisterPoller { } pub async fn poll_and_relay(&mut self) -> Result<(), String> { - //TODO start the timer for the polling iteration let start_polling_instant = tokio::time::Instant::now(); match self.poll_canister().await? { @@ -151,8 +150,10 @@ impl CanisterPoller { PollingStatus::NoMessagesPolled => (), } - //TODO stop the timer for the polling iteration and log the duration - // compute the amout of time to sleep for before polling again + // record the time it took to poll the canister + let delta = start_polling_instant.elapsed(); + histogram!("poller_duration", "canister_id" => self.canister_id.to_string()).record(delta); + let effective_polling_interval = self.compute_effective_polling_interval(start_polling_instant); // if no messages are returned or if the queue is fully drained, sleep for 'effective_polling_interval' before polling again diff --git a/src/ic-websocket-gateway/src/gateway_metrics.rs b/src/ic-websocket-gateway/src/gateway_metrics.rs index 8be7d454..0b6344d2 100644 --- a/src/ic-websocket-gateway/src/gateway_metrics.rs +++ b/src/ic-websocket-gateway/src/gateway_metrics.rs @@ -33,6 +33,10 @@ pub fn init_metrics(address: &str) -> Result<(), Box> { "connection_opening_time", "The time it takes to open a connection" ); + describe_histogram!( + "poller_duration", + "The time it takes to poll the canister" + ); gauge!("active_pollers").set(0.0); diff --git a/telemetry/dashboards/ic_metrics.json b/telemetry/dashboards/ic_metrics.json index 938d6444..59e14eb7 100644 --- a/telemetry/dashboards/ic_metrics.json +++ b/telemetry/dashboards/ic_metrics.json @@ -111,13 +111,221 @@ "title": "Number of active pollers", "type": "gauge" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 14, + "interval": "10s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "poller_duration{quantile=\"1\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{canister_id}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Polling interval by canister id", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average is calculated every 1 minute and aggregated by instance", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 15, + "interval": "10s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg by(instance) (avg_over_time(poller_duration{quantile=\"1\"}[$__interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average polling interval", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 17 }, "id": 9, "panels": [], @@ -193,7 +401,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 18 }, "id": 3, "interval": "10s", @@ -298,7 +506,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 18 }, "id": 4, "interval": "10s", @@ -402,7 +610,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 26 }, "id": 6, "interval": "10s", @@ -508,7 +716,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 26 }, "id": 7, "interval": "10s", @@ -613,7 +821,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "id": 2, "interval": "10s", @@ -717,7 +925,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 34 }, "id": 8, "interval": "10s", @@ -761,7 +969,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 42 }, "id": 11, "panels": [], @@ -833,7 +1041,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 12, "interval": "10s", @@ -936,7 +1144,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 43 }, "id": 13, "interval": "10s", @@ -988,6 +1196,6 @@ "timezone": "browser", "title": "IC Metrics", "uid": "cdfh3d16g8dtsf", - "version": 15, + "version": 16, "weekStart": "" } From d8e42f9b21d623d214e010ba7a79b5b0f4d359cb Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 13:04:30 +0100 Subject: [PATCH 4/7] =?UTF-8?q?refactor=20=E2=9A=99:=20=20Edited=20ic=5Fme?= =?UTF-8?q?trics=20dashboard=20step=20ci=20=F0=9F=9A=80:=20=20Saved=20the?= =?UTF-8?q?=20ic=5Fws=5Fgw=20dashboards=20also=20in=20the=20local=20replic?= =?UTF-8?q?a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- telemetry/dashboards/ic_metrics.json | 6 +- telemetry/dashboards/ic_ws_gw.json | 1962 ++++++++++++++++++++++++++ 2 files changed, 1965 insertions(+), 3 deletions(-) create mode 100644 telemetry/dashboards/ic_ws_gw.json diff --git a/telemetry/dashboards/ic_metrics.json b/telemetry/dashboards/ic_metrics.json index 59e14eb7..6893b2e2 100644 --- a/telemetry/dashboards/ic_metrics.json +++ b/telemetry/dashboards/ic_metrics.json @@ -639,7 +639,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "interval": "5m", + "interval": "1h", "legendFormat": "__auto", "range": true, "refId": "A", @@ -745,7 +745,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "interval": "10m", + "interval": "1d", "legendFormat": "__auto", "range": true, "refId": "A", @@ -1196,6 +1196,6 @@ "timezone": "browser", "title": "IC Metrics", "uid": "cdfh3d16g8dtsf", - "version": 16, + "version": 17, "weekStart": "" } diff --git a/telemetry/dashboards/ic_ws_gw.json b/telemetry/dashboards/ic_ws_gw.json new file mode 100644 index 00000000..c09c314d --- /dev/null +++ b/telemetry/dashboards/ic_ws_gw.json @@ -0,0 +1,1962 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "title": "Proxy Canister", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration of HTTP Canister Proxy", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 16, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "8bcdaa62", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "HTTP Requests", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 15, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "0a5d4dcf", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Client Session" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Past Uptimes of HTTP Canister Proxy", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "panels": [], + "title": "Polling Iterations by Canister ID", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "mqci4-raaaa-aaaao-a2fyq-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration for IC WS Demo", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 17, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "92284c4a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "mqci4-raaaa-aaaao-a2fyq-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Messages from IC WS Demo Canister", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "sswdm-viaaa-aaaap-abugq-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration for sswdm", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 18, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "003150a8", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "sswdm-viaaa-aaaap-abugq-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Canister messages from sswdm", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 2, + "panels": [], + "title": "Duration by Thread ID", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.95, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 45, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "eaaf0a62", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "2" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Client Session" + ], + "valueType": "string" + }, + { + "id": "88e5980f", + "operator": "!=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 2 (excluding HTTP Proxy)", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "b8078415", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "3" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 3", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "d53c2a88", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "4" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 4", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "7e6c2a74", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "5" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 5", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "9d20e8d9", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "6" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 6", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "1fef4771", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "7" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 7", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 4, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "3b6ced2a", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "8" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 8", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 9, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "a43b45ff", + "operator": "=", + "scope": "span", + "tag": "thread.id", + "value": [ + "9" + ], + "valueType": "int" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Accept Connection" + ], + "valueType": "string" + } + ], + "limit": 1000, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Thread ID 9", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "IC WS GW", + "uid": "a360ec48-ff96-448b-9711-e3e80d339722", + "version": 15, + "weekStart": "" +} From 39972417036ca084e3f059a6e4980cbcdfba2521 Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 13:22:10 +0100 Subject: [PATCH 5/7] =?UTF-8?q?docs=20=F0=9F=93=84:=20=20Delete=20the=20ol?= =?UTF-8?q?d=20dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboards/duration_by_thread_id.json | 1047 ----------------- 1 file changed, 1047 deletions(-) delete mode 100644 telemetry/dashboards/duration_by_thread_id.json diff --git a/telemetry/dashboards/duration_by_thread_id.json b/telemetry/dashboards/duration_by_thread_id.json deleted file mode 100644 index 460c7c6d..00000000 --- a/telemetry/dashboards/duration_by_thread_id.json +++ /dev/null @@ -1,1047 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "panels": [], - "title": "Duration by Thread ID", - "type": "row" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "2" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 2", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "3" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 3", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 5, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "4" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 4", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 7, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "5" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 5", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 3, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 6", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 8, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "7" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 7", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 4, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "8" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 8", - "type": "barchart" - }, - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 9, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "Start time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "P214B5B846CF3925F" - }, - "filters": [ - { - "id": "d4ede8da", - "operator": "=", - "scope": "span", - "tag": "thread.id", - "value": [ - "9" - ], - "valueType": "int" - }, - { - "id": "service-name", - "operator": "=", - "scope": "resource", - "tag": "service.name", - "value": [ - "ic-ws-gw-ipc77" - ], - "valueType": "string" - }, - { - "id": "span-name", - "operator": "=", - "scope": "span", - "tag": "name", - "value": [ - "Accept Connection" - ], - "valueType": "string" - } - ], - "limit": 20, - "queryType": "traceqlSearch", - "refId": "A", - "tableType": "traces" - } - ], - "title": "Thread ID 9", - "type": "barchart" - } - ], - "refresh": false, - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Client Connections by Thread ID", - "uid": "a360ec48-ff96-448b-9711-e3e80d339722", - "version": 3, - "weekStart": "" -} - - \ No newline at end of file From 71630e1895374e5059289c1846cd6fb070682927 Mon Sep 17 00:00:00 2001 From: Lorenzo Ronzani Date: Fri, 29 Mar 2024 15:59:26 +0100 Subject: [PATCH 6/7] =?UTF-8?q?refactor=20=E2=9A=99:=20=20Refactored=20acc?= =?UTF-8?q?ording=20the=20provided=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 1 + src/gateway-state/Cargo.toml | 2 +- src/gateway-state/src/lib.rs | 37 +++++++++++-------- src/ic-websocket-gateway/Cargo.toml | 2 +- .../src/canister_poller.rs | 5 ++- .../src/client_session_handler.rs | 8 ++-- .../src/gateway_metrics.rs | 10 ++--- src/ic-websocket-gateway/src/ws_listener.rs | 25 +++++++++---- 8 files changed, 56 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c8e70900..0b0dee54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,3 +26,4 @@ ic-agent = "0.33.0" reqwest = "0.11.7" tokio = { version = "1.29.1", features = ["full"] } tracing = "0.1.40" +metrics = "0.22.1" diff --git a/src/gateway-state/Cargo.toml b/src/gateway-state/Cargo.toml index faf9ec8a..e2905f6d 100644 --- a/src/gateway-state/Cargo.toml +++ b/src/gateway-state/Cargo.toml @@ -11,5 +11,5 @@ dashmap = "5.5.3" ic-agent = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -metrics = "0.22.1" +metrics = { workspace = true } canister-utils = { version = "1.0.0", path = "../canister-utils" } diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs index d3c6fae5..0dd69b95 100644 --- a/src/gateway-state/src/lib.rs +++ b/src/gateway-state/src/lib.rs @@ -1,10 +1,12 @@ -use canister_utils::{ClientKey, IcWsCanisterMessage}; -use dashmap::{mapref::entry::Entry, DashMap}; -use ic_agent::export::Principal; use std::sync::Arc; + +use dashmap::{DashMap, mapref::entry::Entry}; +use ic_agent::export::Principal; +use metrics::gauge; use tokio::sync::mpsc::Sender; use tracing::{debug, Span}; -use metrics::{gauge}; + +use canister_utils::{ClientKey, IcWsCanisterMessage}; /// State of the WS Gateway that can be shared between threads #[derive(Clone)] @@ -57,8 +59,9 @@ impl GatewayState { // Increment the number of clients connected to the canister let clients_connected = poller_state.len(); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // the poller shall not be started again None }, @@ -77,8 +80,9 @@ impl GatewayState { // Increment the number of clients connected to the canister let clients_connected = poller_state.len(); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // the poller shall be started Some(poller_state) }, @@ -112,8 +116,9 @@ impl GatewayState { // Decrement the number of clients connected to the canister let clients_connected = poller_state.len(); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // even if this is the last client session for the canister, do not remove the canister from the gateway state // this will be done by the poller task } @@ -160,8 +165,9 @@ impl GatewayState { Some(_) => { // Decrement the number of clients connected to the canister let clients_connected = poller_state.len(); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); ClientRemovalResult::Removed(client_key) }, @@ -271,14 +277,15 @@ pub type CanisterPrincipal = Principal; #[cfg(test)] mod tests { - use tokio::sync::mpsc::{self, Receiver}; - - use super::*; use std::{ thread, time::{Duration, Instant}, }; + use tokio::sync::mpsc::{self, Receiver}; + + use super::*; + #[tokio::test] async fn should_insert_new_client_channels_and_get_new_poller_state_once() { let clients_count = 1000; diff --git a/src/ic-websocket-gateway/Cargo.toml b/src/ic-websocket-gateway/Cargo.toml index 58935c4d..f096b59c 100644 --- a/src/ic-websocket-gateway/Cargo.toml +++ b/src/ic-websocket-gateway/Cargo.toml @@ -37,7 +37,7 @@ ic-identity = { path = "../ic-identity" } gateway-state = { path = "../gateway-state" } canister-utils = { path = "../canister-utils" } rand = "0.8" -metrics = "0.22.1" +metrics = { workspace = true } metrics-exporter-prometheus = "0.13.1" metrics-util = "0.16.2" diff --git a/src/ic-websocket-gateway/src/canister_poller.rs b/src/ic-websocket-gateway/src/canister_poller.rs index 2279b046..9c107e3e 100644 --- a/src/ic-websocket-gateway/src/canister_poller.rs +++ b/src/ic-websocket-gateway/src/canister_poller.rs @@ -7,8 +7,8 @@ use gateway_state::{ CanisterPrincipal, CanisterRemovalResult, ClientSender, GatewayState, PollerState, }; use ic_agent::{agent::RejectCode, Agent, AgentError}; -use std::{sync::Arc, time::Duration}; use metrics::{gauge, histogram}; +use std::{sync::Arc, time::Duration}; use tokio::{sync::mpsc::Sender, time::timeout}; use tracing::{error, span, trace, warn, Instrument, Level, Span}; @@ -81,7 +81,8 @@ impl CanisterPoller { // register the number of active clients let clients_connected = self.poller_state.len(); - gauge!("clients_connected", "canister_id" => self.canister_id.to_string()).set(clients_connected as f64); + gauge!("clients_connected", "canister_id" => self.canister_id.to_string()) + .set(clients_connected as f64); if let Err(e) = self .poll_and_relay() diff --git a/src/ic-websocket-gateway/src/client_session_handler.rs b/src/ic-websocket-gateway/src/client_session_handler.rs index a44212a9..f205cf3d 100644 --- a/src/ic-websocket-gateway/src/client_session_handler.rs +++ b/src/ic-websocket-gateway/src/client_session_handler.rs @@ -7,9 +7,9 @@ use canister_utils::{ws_close, CanisterWsCloseArguments, ClientKey, IcWsCanister use futures_util::StreamExt; use gateway_state::{CanisterPrincipal, ClientRemovalResult, GatewayState, PollerState}; use ic_agent::Agent; +use metrics::{counter, gauge, histogram}; use std::sync::Arc; use std::time::Instant; -use metrics::{counter, gauge, histogram}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc::{self, Receiver, Sender}, @@ -211,7 +211,8 @@ impl ClientSessionHandler { debug!("Client removed from gateway state"); let delta = client_start_session_time.elapsed(); - histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta); + histogram!("connection_duration", "client_key" => client_key.to_string()) + .record(delta); self.call_ws_close(&canister_id, client_key).await; @@ -245,7 +246,8 @@ impl ClientSessionHandler { debug!("Client removed from gateway state"); let delta = client_start_session_time.elapsed(); - histogram!("connection_duration", "client_key" => client_key.to_string()).record(delta); + histogram!("connection_duration", "client_key" => client_key.to_string()) + .record(delta); self.call_ws_close(&canister_id, client_key).await; diff --git a/src/ic-websocket-gateway/src/gateway_metrics.rs b/src/ic-websocket-gateway/src/gateway_metrics.rs index 0b6344d2..aad3ea5b 100644 --- a/src/ic-websocket-gateway/src/gateway_metrics.rs +++ b/src/ic-websocket-gateway/src/gateway_metrics.rs @@ -15,7 +15,10 @@ pub fn init_metrics(address: &str) -> Result<(), Box> { .install() .expect("failed to install Prometheus recorder"); - describe_counter!("client_connected_count", "Each time that a client connects it emits a point"); + describe_counter!( + "client_connected_count", + "Each time that a client connects it emits a point" + ); describe_gauge!( "clients_connected", "The number of clients currently connected" @@ -33,10 +36,7 @@ pub fn init_metrics(address: &str) -> Result<(), Box> { "connection_opening_time", "The time it takes to open a connection" ); - describe_histogram!( - "poller_duration", - "The time it takes to poll the canister" - ); + describe_histogram!("poller_duration", "The time it takes to poll the canister"); gauge!("active_pollers").set(0.0); diff --git a/src/ic-websocket-gateway/src/ws_listener.rs b/src/ic-websocket-gateway/src/ws_listener.rs index a2f21801..a00e12b3 100644 --- a/src/ic-websocket-gateway/src/ws_listener.rs +++ b/src/ic-websocket-gateway/src/ws_listener.rs @@ -1,11 +1,11 @@ use crate::client_session_handler::ClientSessionHandler; use gateway_state::GatewayState; use ic_agent::Agent; +use metrics::histogram; use native_tls::Identity; -use std::{fs, net::SocketAddr, sync::Arc, time::Duration}; use std::collections::HashMap; use std::time::Instant; -use metrics::{histogram}; +use std::{fs, net::SocketAddr, sync::Arc, time::Duration}; use tokio::{ net::{TcpListener, TcpStream}, select, @@ -158,7 +158,8 @@ impl WsListener { let client_id = self.next_client_id; let tls_acceptor = self.tls_acceptor.clone(); - self.clients_connection_time.insert(client_id, Instant::now()); + self.clients_connection_time + .insert(client_id, Instant::now()); tokio::spawn( async move { @@ -194,7 +195,8 @@ impl WsListener { .expect("ws listener's side of the channel dropped"); let delta = start.elapsed(); - histogram!("tls_resolution_time", "client_id" => client_id.to_string()).record(delta); + histogram!("tls_resolution_time", "client_id" => client_id.to_string()) + .record(delta); }, Err(e) => { error!("Failed to accept connection: {:?}", e); @@ -216,12 +218,21 @@ impl WsListener { let polling_interval_ms = self.polling_interval_ms; // spawn a session handler task for each incoming client connection - let start = self.clients_connection_time.get(&client_id).unwrap().clone(); + let start = self + .clients_connection_time + .get(&client_id) + .unwrap() + .clone(); tokio::spawn( async move { - let mut client_session_handler = - ClientSessionHandler::new(client_id, agent, gateway_state, polling_interval_ms, start); + let mut client_session_handler = ClientSessionHandler::new( + client_id, + agent, + gateway_state, + polling_interval_ms, + start, + ); debug!("Started client session handler task"); if let Err(e) = { From af16b75961825fd212bfd4a2ef7b5f8302348b78 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Fri, 29 Mar 2024 16:43:04 +0100 Subject: [PATCH 7/7] chore: cargo fmt --- src/gateway-state/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs index 39f6f9e2..ddd2bcea 100644 --- a/src/gateway-state/src/lib.rs +++ b/src/gateway-state/src/lib.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use dashmap::{DashMap, mapref::entry::Entry}; +use dashmap::{mapref::entry::Entry, DashMap}; use ic_agent::export::Principal; use metrics::gauge; use tokio::sync::mpsc::Sender;