diff --git a/cmd/ethrex/initializers.rs b/cmd/ethrex/initializers.rs index d010cad165f..163071df6d1 100644 --- a/cmd/ethrex/initializers.rs +++ b/cmd/ethrex/initializers.rs @@ -11,6 +11,7 @@ use ethrex_common::types::Genesis; use ethrex_config::networks::Network; use ethrex_metrics::profiling::{FunctionProfilingLayer, initialize_block_processing_profile}; +use ethrex_metrics::rpc::initialize_rpc_metrics; use ethrex_p2p::rlpx::initiator::RLPxInitiator; use ethrex_p2p::{ discv4::peer_table::PeerTable, @@ -89,6 +90,7 @@ pub fn init_metrics(opts: &Options, tracker: TaskTracker) { ); initialize_block_processing_profile(); + initialize_rpc_metrics(); tracker.spawn(metrics_api); } diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index c740f09bcf9..d589b34a5a8 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -1,9 +1,8 @@ use axum::{Router, routing::get}; -use crate::profiling::gather_profiling_metrics; - use crate::{ - MetricsApiError, blocks::METRICS_BLOCKS, process::METRICS_PROCESS, transactions::METRICS_TX, + MetricsApiError, blocks::METRICS_BLOCKS, gather_default_metrics, process::METRICS_PROCESS, + transactions::METRICS_TX, }; pub async fn start_prometheus_metrics_api( @@ -32,10 +31,10 @@ pub(crate) async fn get_metrics() -> String { }; ret_string.push('\n'); - match gather_profiling_metrics() { + match gather_default_metrics() { Ok(string) => ret_string.push_str(&string), Err(_) => { - tracing::error!("Failed to register METRICS_PROFILING"); + tracing::error!("Failed to gather default Prometheus metrics"); return String::new(); } }; diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index 8047c3be413..40b0c1ab89e 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -8,6 +8,8 @@ pub mod l2; pub mod process; #[cfg(feature = "api")] pub mod profiling; +#[cfg(feature = "api")] +pub mod rpc; #[cfg(any(feature = "api", feature = "transactions"))] pub mod transactions; @@ -70,3 +72,24 @@ pub enum MetricsError { #[error("MetricsL2Error {0}")] FromUtf8Error(#[from] std::string::FromUtf8Error), } + +#[cfg(feature = "api")] +/// Returns all metrics currently registered in Prometheus' default registry. +/// +/// Both profiling and RPC metrics register with this default registry, and the +/// metrics API surfaces them by calling this helper. +pub fn gather_default_metrics() -> Result { + use prometheus::{Encoder, TextEncoder}; + + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; + + Ok(res) +} diff --git a/crates/blockchain/metrics/profiling.rs b/crates/blockchain/metrics/profiling.rs index dc746191f37..a82c677bb8e 100644 --- a/crates/blockchain/metrics/profiling.rs +++ b/crates/blockchain/metrics/profiling.rs @@ -1,5 +1,5 @@ -use prometheus::{Encoder, HistogramTimer, HistogramVec, TextEncoder, register_histogram_vec}; -use std::{future::Future, sync::LazyLock}; +use prometheus::{HistogramTimer, HistogramVec, register_histogram_vec}; +use std::sync::LazyLock; use tracing::{ Subscriber, field::{Field, Visit}, @@ -7,11 +7,12 @@ use tracing::{ }; use tracing_subscriber::{Layer, layer::Context, registry::LookupSpan}; -use crate::MetricsError; - pub static METRICS_BLOCK_PROCESSING_PROFILE: LazyLock = LazyLock::new(initialize_histogram_vec); +// Metrics defined in this module register into the Prometheus default registry. +// The metrics API exposes them by calling `gather_default_metrics()`. + fn initialize_histogram_vec() -> HistogramVec { register_histogram_vec!( "function_duration_seconds", @@ -111,45 +112,6 @@ where } } -/// Records the duration of an async operation in the function profiling histogram. -/// -/// This provides a lightweight alternative to the `#[instrument]` attribute when you need -/// manual control over timing instrumentation, such as in RPC handlers. -/// -/// # Parameters -/// * `namespace` - Category for the metric (e.g., "rpc", "engine", "block_execution") -/// * `function_name` - Name identifier for the operation being timed -/// * `future` - The async operation to time -/// -/// Use this function when you need to instrument an async operation for duration metrics, -/// but cannot or do not want to use the `#[instrument]` attribute (for example, in RPC handlers). -pub async fn record_async_duration(namespace: &str, function_name: &str, future: Fut) -> T -where - Fut: Future, -{ - let timer = METRICS_BLOCK_PROCESSING_PROFILE - .with_label_values(&[namespace, function_name]) - .start_timer(); - - let output = future.await; - timer.observe_duration(); - output -} - -pub fn gather_profiling_metrics() -> Result { - let encoder = TextEncoder::new(); - let metric_families = prometheus::gather(); - - let mut buffer = Vec::new(); - encoder - .encode(&metric_families, &mut buffer) - .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; - - let res = String::from_utf8(buffer)?; - - Ok(res) -} - pub fn initialize_block_processing_profile() { METRICS_BLOCK_PROCESSING_PROFILE.reset(); } diff --git a/crates/blockchain/metrics/rpc.rs b/crates/blockchain/metrics/rpc.rs new file mode 100644 index 00000000000..97bc4951277 --- /dev/null +++ b/crates/blockchain/metrics/rpc.rs @@ -0,0 +1,85 @@ +use prometheus::{CounterVec, HistogramVec, register_counter_vec, register_histogram_vec}; +use std::{future::Future, sync::LazyLock}; + +pub static METRICS_RPC_REQUEST_OUTCOMES: LazyLock = + LazyLock::new(initialize_rpc_outcomes_counter); + +pub static METRICS_RPC_DURATION: LazyLock = + LazyLock::new(initialize_rpc_duration_histogram); + +// Metrics defined in this module register into the Prometheus default registry. +// The metrics API exposes them by calling `gather_default_metrics()`. + +fn initialize_rpc_outcomes_counter() -> CounterVec { + register_counter_vec!( + "rpc_requests_total", + "Total number of RPC requests partitioned by namespace, method, and outcome", + &["namespace", "method", "outcome", "error_kind"], + ) + .unwrap() +} + +fn initialize_rpc_duration_histogram() -> HistogramVec { + register_histogram_vec!( + "rpc_request_duration_seconds", + "Histogram of RPC request handling duration partitioned by namespace and method", + &["namespace", "method"], + ) + .unwrap() +} + +/// Represents the outcome of an RPC request when recording metrics. +#[derive(Clone)] +pub enum RpcOutcome { + Success, + Error(&'static str), +} + +impl RpcOutcome { + fn as_label(&self) -> &'static str { + match self { + RpcOutcome::Success => "success", + RpcOutcome::Error(_) => "error", + } + } + + fn error_kind(&self) -> &str { + match self { + RpcOutcome::Success => "", + RpcOutcome::Error(kind) => kind, + } + } +} + +pub fn record_rpc_outcome(namespace: &str, method: &str, outcome: RpcOutcome) { + METRICS_RPC_REQUEST_OUTCOMES + .with_label_values(&[namespace, method, outcome.as_label(), outcome.error_kind()]) + .inc(); +} + +pub fn initialize_rpc_metrics() { + METRICS_RPC_REQUEST_OUTCOMES.reset(); + METRICS_RPC_DURATION.reset(); +} + +/// Records the duration of an async operation in the RPC request duration histogram. +/// +/// This provides a lightweight alternative to the `#[instrument]` attribute. +/// +/// # Parameters +/// * `namespace` - Category for the metric (e.g., "rpc", "engine", "block_execution") +/// * `method` - Name identifier for the operation being timed +/// * `future` - The async operation to time +/// +pub async fn record_async_duration(namespace: &str, method: &str, future: Fut) -> T +where + Fut: Future, +{ + let timer = METRICS_RPC_DURATION + .with_label_values(&[namespace, method]) + .start_timer(); + + let output = future.await; + timer.observe_duration(); + output +} diff --git a/crates/networking/rpc/rpc.rs b/crates/networking/rpc/rpc.rs index 15ae70c5ea9..f0da5877f1b 100644 --- a/crates/networking/rpc/rpc.rs +++ b/crates/networking/rpc/rpc.rs @@ -55,7 +55,7 @@ use bytes::Bytes; use ethrex_blockchain::Blockchain; use ethrex_blockchain::error::ChainError; use ethrex_common::types::Block; -use ethrex_metrics::profiling::record_async_duration; +use ethrex_metrics::rpc::{RpcOutcome, record_async_duration, record_rpc_outcome}; use ethrex_p2p::peer_handler::PeerHandler; use ethrex_p2p::sync_manager::SyncManager; use ethrex_p2p::types::Node; @@ -196,16 +196,48 @@ pub trait RpcHandler: Sized { Ok(RpcNamespace::Engine) => "engine", _ => "rpc", }; + let method = req.method.as_str(); + + let result = + record_async_duration( + namespace, + method, + async move { request.handle(context).await }, + ) + .await; + + let outcome = match &result { + Ok(_) => RpcOutcome::Success, + Err(err) => RpcOutcome::Error(get_error_kind(err)), + }; + record_rpc_outcome(namespace, method, outcome); - record_async_duration(namespace, req.method.as_str(), async move { - request.handle(context).await - }) - .await + result } async fn handle(&self, context: RpcApiContext) -> Result; } +fn get_error_kind(err: &RpcErr) -> &'static str { + match err { + RpcErr::MethodNotFound(_) => "MethodNotFound", + RpcErr::WrongParam(_) => "WrongParam", + RpcErr::BadParams(_) => "BadParams", + RpcErr::MissingParam(_) => "MissingParam", + RpcErr::TooLargeRequest => "TooLargeRequest", + RpcErr::BadHexFormat(_) => "BadHexFormat", + RpcErr::UnsuportedFork(_) => "UnsuportedFork", + RpcErr::Internal(_) => "Internal", + RpcErr::Vm(_) => "Vm", + RpcErr::Revert { .. } => "Revert", + RpcErr::Halt { .. } => "Halt", + RpcErr::AuthenticationError(_) => "AuthenticationError", + RpcErr::InvalidForkChoiceState(_) => "InvalidForkChoiceState", + RpcErr::InvalidPayloadAttributes(_) => "InvalidPayloadAttributes", + RpcErr::UnknownPayload(_) => "UnknownPayload", + } +} + pub const FILTER_DURATION: Duration = { if cfg!(test) { Duration::from_secs(1) diff --git a/docs/developers/l1/dashboards.md b/docs/developers/l1/dashboards.md index 7b9316c3af4..27a1d68a991 100644 --- a/docs/developers/l1/dashboards.md +++ b/docs/developers/l1/dashboards.md @@ -94,16 +94,21 @@ Collapsed row that surfaces the `namespace="engine"` Prometheus timers so you ca ![Engine API row](img/engine_api_row.png) -### Engine Request Rate by Method -Shows how many Engine API calls per second we process, split by JSON-RPC method and averaged across the currently selected dashboard range. +### Engine Total Time per Method +Pie chart that shows where Engine time is spent across methods over the selected range. Quickly surfaces which endpoints dominate total processing time. -![Engine Request Rate by Method](img/engine_request_rate_by_method.png) +![Engine Total Time per Method](img/engine_total_time_per_method.png) ### Engine Latency by Methods (Avg Duration) Bar gauge of the historical average latency per Engine method over the selected time range. ![Engine Latency by Methods](img/engine_latency_by_methods.png) +### Engine Request Rate by Method +Shows how many Engine API calls per second we process, split by JSON-RPC method and averaged across the currently selected dashboard range. + +![Engine Request Rate by Method](img/engine_request_rate_by_method.png) + ### Engine Latency by Method Live timeseries that tries to correlate to the per-block execution time by showing real-time latency per Engine method with an 18 s lookback window. @@ -117,10 +122,10 @@ Another collapsed row focused on the public JSON-RPC surface (`namespace="rpc"`) ![RPC API row](img/rpc_api_row.png) -### RPC Time per Method +### RPC Total Time per Method Pie chart that shows where RPC time is spent across methods over the selected range. Quickly surfaces which endpoints dominate total processing time. -![RPC Time per Method](img/rpc_time_per_method.png) +![RPC Total Time per Method](img/rpc_total_time_per_method.png) ### Slowest RPC Methods Table listing the highest average-latency methods over the active dashboard range. Used to prioritise optimisation or caching efforts. @@ -139,6 +144,28 @@ Live timeseries that tries to correlate to the per-block execution time by showi _**Limitations**: The RPC latency views inherit the same windowing caveats as the Engine charts: averages use the dashboard time range while the live chart relies on an 18 s window._ +## Engine and RPC Error rates + +Collapsed row showing error rates for both Engine and RPC APIs side by side and a deagreagated panel by method and kind of error. Each panel repeats per instance to be able to compare behaviour across nodes. + +![Engine and RPC Error rates row](img/engine_and_rpc_error_rates_row.png) + +### Engine Success/Error Rate +Shows the rate of successful vs. failed Engine API requests per second. + +![Engine Success/Error Rate](img/engine_success_error_rate.png) + +### RPC Success/Error Rate +Shows the rate of successful vs. failed RPC API requests per second. + +![RPC Success/Error Rate](img/rpc_success_error_rate.png) + +### Engine and RPC Errors % by Method and Kind + +Deaggregated view of error percentages split by method and error kind for both Engine and RPC APIs. The % are calculated against total requests for a particular method, so all different error percentage for a method should sum up to the percentage of errors for that method. + +![Engine and RPC Errors % by Method and Kind](img/engine_and_rpc_errors_by_method_and_kind.png) + ## Process and server info Row panels showing process-level and host-level metrics to help you monitor resource usage and spot potential issues. diff --git a/docs/developers/l1/img/engine_and_rpc_error_rates_row.png b/docs/developers/l1/img/engine_and_rpc_error_rates_row.png new file mode 100644 index 00000000000..e1b514d03ad Binary files /dev/null and b/docs/developers/l1/img/engine_and_rpc_error_rates_row.png differ diff --git a/docs/developers/l1/img/engine_and_rpc_errors_by_method_and_kind.png b/docs/developers/l1/img/engine_and_rpc_errors_by_method_and_kind.png new file mode 100644 index 00000000000..b0530c2e887 Binary files /dev/null and b/docs/developers/l1/img/engine_and_rpc_errors_by_method_and_kind.png differ diff --git a/docs/developers/l1/img/engine_api_row.png b/docs/developers/l1/img/engine_api_row.png index a77024cfb25..5cecbea776c 100644 Binary files a/docs/developers/l1/img/engine_api_row.png and b/docs/developers/l1/img/engine_api_row.png differ diff --git a/docs/developers/l1/img/engine_success_error_rate.png b/docs/developers/l1/img/engine_success_error_rate.png new file mode 100644 index 00000000000..b5955bc5bc1 Binary files /dev/null and b/docs/developers/l1/img/engine_success_error_rate.png differ diff --git a/docs/developers/l1/img/engine_total_time_per_method.png b/docs/developers/l1/img/engine_total_time_per_method.png new file mode 100644 index 00000000000..5e7c88745c9 Binary files /dev/null and b/docs/developers/l1/img/engine_total_time_per_method.png differ diff --git a/docs/developers/l1/img/rpc_api_row.png b/docs/developers/l1/img/rpc_api_row.png index 0bd5ae99edc..2fa0b4d5a2a 100644 Binary files a/docs/developers/l1/img/rpc_api_row.png and b/docs/developers/l1/img/rpc_api_row.png differ diff --git a/docs/developers/l1/img/rpc_success_error_rate.png b/docs/developers/l1/img/rpc_success_error_rate.png new file mode 100644 index 00000000000..1b6b4940120 Binary files /dev/null and b/docs/developers/l1/img/rpc_success_error_rate.png differ diff --git a/docs/developers/l1/img/rpc_time_per_method.png b/docs/developers/l1/img/rpc_time_per_method.png deleted file mode 100644 index f41c7c5b6ec..00000000000 Binary files a/docs/developers/l1/img/rpc_time_per_method.png and /dev/null differ diff --git a/docs/developers/l1/img/rpc_total_time_per_method.png b/docs/developers/l1/img/rpc_total_time_per_method.png new file mode 100644 index 00000000000..3aa49aa7a30 Binary files /dev/null and b/docs/developers/l1/img/rpc_total_time_per_method.png differ diff --git a/metrics/provisioning/grafana/dashboards/common_dashboards/ethrex_l1_perf.json b/metrics/provisioning/grafana/dashboards/common_dashboards/ethrex_l1_perf.json index b450de32bca..dadfdc85dc2 100644 --- a/metrics/provisioning/grafana/dashboards/common_dashboards/ethrex_l1_perf.json +++ b/metrics/provisioning/grafana/dashboards/common_dashboards/ethrex_l1_perf.json @@ -1212,7 +1212,7 @@ "h": 13, "w": 6, "x": 0, - "y": 27 + "y": 83 }, "id": 18, "options": { @@ -1325,7 +1325,7 @@ "h": 13, "w": 6, "x": 6, - "y": 27 + "y": 83 }, "id": 63, "options": { @@ -1453,7 +1453,7 @@ "h": 13, "w": 12, "x": 12, - "y": 27 + "y": 83 }, "id": 61, "interval": "5s", @@ -1597,84 +1597,54 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Total time spent per method for all RPC endpoints", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 14, - "w": 7, + "w": 5, "x": 0, - "y": 28 + "y": 97 }, - "id": 113, + "id": 118, "options": { + "displayLabels": [], "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { "calcs": [ - "mean", - "max" + "sum" ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "fields": "", + "values": false }, + "sort": "desc", "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "12.2.1", @@ -1687,15 +1657,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (function_name) (\n rate(function_duration_seconds_count{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\",\n namespace=\"engine\"\n }[$__range])\n)", - "instant": false, - "legendFormat": "{{function_name}}", - "range": true, + "expr": "sum by (method) (increase(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range]))", + "instant": true, + "legendFormat": "{{method}}", + "range": false, "refId": "A" } ], - "title": "Engine Request Rate by Method - $instance", - "type": "timeseries" + "title": "Engine Total Time per Method - $instance", + "type": "piechart" }, { "datasource": { @@ -1728,9 +1698,9 @@ }, "gridPos": { "h": 14, - "w": 5, - "x": 7, - "y": 28 + "w": 3, + "x": 5, + "y": 97 }, "id": 115, "interval": "10s", @@ -1769,7 +1739,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (function_name) (increase(function_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range])) \n/ sum by (function_name) (increase(function_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range]))", + "expr": "sum by (method) (increase(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range])) \n/ sum by (method) (increase(rpc_request_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range]))", "instant": false, "interval": "", "legendFormat": "__auto", @@ -1780,6 +1750,107 @@ "title": "Engine Latency by Methods (AVG Duration) - $instance", "type": "bargauge" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 4, + "x": 8, + "y": 97 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "repeat": "instance", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (\n rate(rpc_request_duration_seconds_count{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\",\n namespace=\"engine\"\n }[$__range])\n)", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Engine Request Rate by Method - $instance", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -1846,7 +1917,7 @@ "h": 14, "w": 12, "x": 12, - "y": 28 + "y": 97 }, "id": 112, "interval": "10s", @@ -1877,7 +1948,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (function_name) (increase(function_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[18s])) \n/ sum by (function_name) (increase(function_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[18s]))", + "expr": "sum by (method) (increase(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[18s])) \n/ sum by (method) (increase(rpc_request_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[18s]))", "instant": false, "interval": "", "legendFormat": "__auto", @@ -1907,6 +1978,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Total time spent per method for all RPC endpoints", "fieldConfig": { "defaults": { "color": { @@ -1928,7 +2000,7 @@ "h": 14, "w": 7, "x": 0, - "y": 29 + "y": 112 }, "id": 103, "options": { @@ -1966,14 +2038,14 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (function_name) (increase(function_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]))", + "expr": "sum by (method) (increase(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]))", "instant": true, - "legendFormat": "{{function_name}}", + "legendFormat": "{{method}}", "range": false, "refId": "A" } ], - "title": "RPC Average Time per Method - $instance", + "title": "RPC Total Time per Method - $instance", "type": "piechart" }, { @@ -2026,7 +2098,7 @@ "properties": [ { "id": "custom.width", - "value": 300 + "value": 214 } ] } @@ -2034,9 +2106,9 @@ }, "gridPos": { "h": 14, - "w": 5, + "w": 3, "x": 7, - "y": 29 + "y": 112 }, "id": 104, "options": { @@ -2059,7 +2131,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(function_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]) / rate(function_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range])", + "expr": "rate(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]) / rate(rpc_request_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range])", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2081,7 +2153,7 @@ "indexByName": {}, "renameByName": { "Value": "Avg Duration", - "function_name": "Method" + "method": "Method" } } } @@ -2148,9 +2220,9 @@ }, "gridPos": { "h": 14, - "w": 6, - "x": 12, - "y": 29 + "w": 5, + "x": 10, + "y": 112 }, "id": 101, "options": { @@ -2181,8 +2253,8 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (function_name) (rate(function_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]))", - "legendFormat": "{{function_name}}", + "expr": "sum by (method) (rate(rpc_request_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]))", + "legendFormat": "{{method}}", "range": true, "refId": "A" } @@ -2250,9 +2322,9 @@ }, "gridPos": { "h": 14, - "w": 6, - "x": 18, - "y": 29 + "w": 9, + "x": 15, + "y": 112 }, "id": 102, "options": { @@ -2281,31 +2353,700 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (function_name) (increase(function_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[18s])) \n/ sum by (function_name) (increase(function_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[18s]))", - "legendFormat": "{{ function_name }}", + "expr": "sum by (method) (increase(rpc_request_duration_seconds_sum{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[18s])) \n/ sum by (method) (increase(rpc_request_duration_seconds_count{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[18s]))", + "legendFormat": "{{ method }}", "range": true, "refId": "A" } ], - "title": "RPC Latency by Methods - $instance", - "type": "timeseries" + "title": "RPC Latency by Methods - $instance", + "type": "timeseries" + } + ], + "title": "RPC API", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 117, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Requests/sec", + "axisPlacement": "auto", + "axisSoftMax": 0.3, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 0, + "y": 86 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "repeat": "instance", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(rpc_requests_total{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"engine\"}[$__range]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Engine Success/Error Rate - $instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Requests/sec", + "axisPlacement": "auto", + "axisSoftMax": 0.7, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 6, + "y": 86 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "repeat": "instance", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(rpc_requests_total{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\", namespace=\"rpc\"}[$__range]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Success/Error Rate - $instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This shows error % by kind in the total requests for a particular method, ie:\n- 50 request to method_a\n- 5 internal errors on method_a\n- 12 timeout errors on method_a\n\nResult:\n- 10% of method_a - % of internal error\n- 24% of method_a - % of timeout error ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Requests", + "axisPlacement": "right", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*%/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.axisLabel", + "value": "Error %" + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "engine_getPayloadBodiesByRangeV1 - % of Internal error" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 86 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "repeat": "instance", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method, error_kind) (\n increase(rpc_requests_total{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\",\n outcome=\"error\"\n }[$__range])\n)", + "hide": true, + "instant": false, + "legendFormat": "{{method}} - {{error_kind}} error", + "range": true, + "refId": "errors_by_method_kind" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (\n increase(rpc_requests_total{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\"\n }[$__range])\n)", + "hide": true, + "instant": false, + "legendFormat": "{{method}} - Total requests", + "range": true, + "refId": "total_by_method" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 *\nsum by (method, error_kind) (\n increase(rpc_requests_total{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\",\n outcome=\"error\"\n }[$__range])\n)\n/\non (method) group_left\nsum by (method) (\n increase(rpc_requests_total{\n job=\"$job\",\n instance=~\"$instance(:\\\\d+)?$\"\n }[$__range])\n)", + "hide": false, + "instant": false, + "legendFormat": "{{method}} - % of {{error_kind}} error", + "range": true, + "refId": "error_pct_by_method_kind" + } + ], + "title": "Engine & RPC Errors % by Method & Kind - $instance", + "type": "timeseries" + } + ], + "title": "Engine and RPC Error rates", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 48, + "panels": [], + "title": "Process & Server Info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "process_open_fds{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\"}", + "legendFormat": "{{ instance }} FDs used", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs Historic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This calculates the size in bytes of the datadir path where the DB is stored.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" } - ], - "title": "RPC API", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 }, - "id": 48, - "panels": [], - "title": "Process & Server Info", - "type": "row" + "pluginVersion": "12.2.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "datadir_size_bytes{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{ instance }} Datadir Size (GB)", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Datadir Size", + "type": "timeseries" }, { "datasource": { @@ -2339,7 +3080,7 @@ "h": 3, "w": 5, "x": 0, - "y": 30 + "y": 36 }, "id": 34, "options": { @@ -2408,7 +3149,7 @@ "h": 3, "w": 2, "x": 5, - "y": 30 + "y": 36 }, "id": 33, "options": { @@ -2476,7 +3217,7 @@ "h": 3, "w": 5, "x": 7, - "y": 30 + "y": 36 }, "id": 43, "options": { @@ -2513,216 +3254,6 @@ "title": "Open FDs", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This calculates the size in bytes of the datadir path where the DB is stored.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 46, - "options": { - "legend": { - "calcs": [ - "max", - "mean", - "last" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "datadir_size_bytes{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{ instance }} Datadir Size (GB)", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Datadir Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "red", - "value": 85 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "process_open_fds{job=\"$job\", instance=~\"$instance(:\\\\d+)?$\"}", - "legendFormat": "{{ instance }} FDs used", - "range": true, - "refId": "A" - } - ], - "title": "Open FDs Historic", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -2790,7 +3321,7 @@ "h": 6, "w": 12, "x": 0, - "y": 38 + "y": 39 }, "id": 30, "options": { @@ -2918,7 +3449,7 @@ "h": 6, "w": 12, "x": 12, - "y": 38 + "y": 39 }, "id": 31, "options": { @@ -3022,7 +3553,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 45 }, "id": 45, "options": { @@ -3161,7 +3692,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 45 }, "id": 42, "options": { @@ -3211,7 +3742,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 55, "panels": [ @@ -3281,7 +3812,7 @@ "h": 8, "w": 6, "x": 0, - "y": 1438 + "y": 388 }, "id": 56, "options": { @@ -3301,7 +3832,7 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", @@ -3380,7 +3911,7 @@ "h": 8, "w": 9, "x": 6, - "y": 1438 + "y": 388 }, "id": 57, "options": { @@ -3400,7 +3931,7 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", @@ -3480,7 +4011,7 @@ "h": 8, "w": 9, "x": 15, - "y": 1438 + "y": 388 }, "id": 58, "options": { @@ -3500,7 +4031,7 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.2.1", "targets": [ { "editorMode": "code", @@ -3602,5 +4133,5 @@ "timezone": "utc", "title": "Ethrex L1 - Perf Dashboard", "uid": "beoru4vp59yiof", - "version": 39 + "version": 42 }