From 5bc3b009c688575c41b0272e5db9a3a84ca37001 Mon Sep 17 00:00:00 2001 From: liranmauda Date: Wed, 1 Feb 2023 15:03:52 +0200 Subject: [PATCH 1/2] Removing read and write error count for NSFS I/O metrics - Removing read and write error count for NSFS I/O metrics - if namespace count does have some error prompting a warning. just that we will be aware and consider re-adding it Signed-off-by: liranmauda --- src/sdk/endpoint_stats_collector.js | 23 ++++++++++++++----- .../system_services/stats_aggregator.js | 10 +++----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/sdk/endpoint_stats_collector.js b/src/sdk/endpoint_stats_collector.js index 92d08fb2e5..57d1ef94f5 100644 --- a/src/sdk/endpoint_stats_collector.js +++ b/src/sdk/endpoint_stats_collector.js @@ -44,7 +44,7 @@ class EndpointStatsCollector { this.op_stats = {}; this.fs_workers_stats = {}; this.reset_all_stats(); - this.nsfs_io_counters = this._new_namespace_stats(); + this.nsfs_io_counters = this._new_namespace_nsfs_stats(); this.prom_metrics_report = prom_report.get_endpoint_report(); } @@ -59,7 +59,7 @@ class EndpointStatsCollector { } reset_all_nsfs_stats() { - this.nsfs_io_counters = this._new_namespace_stats(); + this.nsfs_io_counters = this._new_namespace_nsfs_stats(); this.op_stats = {}; this.fs_workers_stats = {}; } @@ -115,6 +115,15 @@ class EndpointStatsCollector { }; } + _new_namespace_nsfs_stats() { + return { + read_count: 0, + write_count: 0, + read_bytes: 0, + write_bytes: 0, + }; + } + update_namespace_read_stats({ namespace_resource_id, bucket_name, size = 0, count = 0, is_err }) { this.namespace_stats[namespace_resource_id] = this.namespace_stats[namespace_resource_id] || this._new_namespace_stats(); const io_stats = this.namespace_stats[namespace_resource_id]; @@ -218,8 +227,9 @@ class EndpointStatsCollector { update_nsfs_read_counters({ size = 0, count = 0, is_err }) { if (is_err) { - this.nsfs_io_counters.error_read_count += count; - this.nsfs_io_counters.error_read_bytes += size; + dbg.warn(`unexpectedly reached here upon error, we need to figure out why and maybe re-add the error counters`); + // this.nsfs_io_counters.error_read_count += count; + // this.nsfs_io_counters.error_read_bytes += size; } else { this.nsfs_io_counters.read_count += count; this.nsfs_io_counters.read_bytes += size; @@ -233,8 +243,9 @@ class EndpointStatsCollector { update_nsfs_write_counters({ size = 0, count = 0, is_err }) { if (is_err) { - this.nsfs_io_counters.error_write_count += count; - this.nsfs_io_counters.error_write_bytes += size; + dbg.warn(`unexpectedly reached here upon error, we need to figure out why and maybe re-add the error counters`); + // this.nsfs_io_counters.error_write_count += count; + // this.nsfs_io_counters.error_write_bytes += size; } else { this.nsfs_io_counters.write_count += count; this.nsfs_io_counters.write_bytes += size; diff --git a/src/server/system_services/stats_aggregator.js b/src/server/system_services/stats_aggregator.js index f38e95ba5d..8b1edf4e2b 100644 --- a/src/server/system_services/stats_aggregator.js +++ b/src/server/system_services/stats_aggregator.js @@ -47,7 +47,7 @@ let last_partial_stats_requested = 0; const PARTIAL_STATS_REQUESTED_GRACE_TIME = 30 * 1000; // Will hold the nsfs counters/metrics -let nsfs_io_counters = _new_nsfs_stats(); +let nsfs_io_counters = _new_namespace_nsfs_stats(); // Will hold the op stats (op name, min/max/avg time, count, error count) let op_stats = {}; let fs_workers_stats = {}; @@ -1344,23 +1344,19 @@ function _set_fs_workers_stats(fsworker_name, stats) { } } -function _new_nsfs_stats() { +function _new_namespace_nsfs_stats() { return { read_count: 0, write_count: 0, read_bytes: 0, write_bytes: 0, - error_write_bytes: 0, - error_write_count: 0, - error_read_bytes: 0, - error_read_count: 0, }; } // Will return the current nsfs_io_counters and reset it. function get_nsfs_io_stats() { const nsfs_io_stats = nsfs_io_counters; - nsfs_io_counters = _new_nsfs_stats(); + nsfs_io_counters = _new_namespace_nsfs_stats(); return nsfs_io_stats; } From 30212967a482f93e1c3184fc4faace924b929774 Mon Sep 17 00:00:00 2001 From: liranmauda Date: Wed, 1 Feb 2023 16:36:55 +0200 Subject: [PATCH 2/2] Adding to the nsfs metrics report the metric type (fs/op/io) - Adding to the nsfs metrics report the metric type (fs/op/io) Signed-off-by: liranmauda --- src/server/web_server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/web_server.js b/src/server/web_server.js index 5281ca9c2b..0427540247 100755 --- a/src/server/web_server.js +++ b/src/server/web_server.js @@ -300,7 +300,7 @@ function _create_nsfs_report() { const nsfs_counters = stats_aggregator.get_nsfs_io_stats(); // Building the report per io and value for (const [key, value] of Object.entries(nsfs_counters)) { - const metric = `NooBaa_nsfs_${key}`.toLowerCase(); + const metric = `noobaa_nsfs_io_${key}`.toLowerCase(); nsfs_report += `${metric}: ${value}
`; } @@ -309,7 +309,7 @@ function _create_nsfs_report() { for (const [op_name, obj] of Object.entries(op_stats)) { nsfs_report += `
`; for (const [key, value] of Object.entries(obj)) { - const metric = `NooBaa_nsfs_${op_name}_${key}`.toLowerCase(); + const metric = `noobaa_nsfs_op_${op_name}_${key}`.toLowerCase(); nsfs_report += `${metric}: ${value}
`; } } @@ -319,7 +319,7 @@ function _create_nsfs_report() { for (const [fs_worker_name, obj] of Object.entries(fs_workers_stats)) { nsfs_report += `
`; for (const [key, value] of Object.entries(obj)) { - const metric = `NooBaa_nsfs_${fs_worker_name}_${key}`.toLowerCase(); + const metric = `noobaa_nsfs_fs_${fs_worker_name}_${key}`.toLowerCase(); nsfs_report += `${metric}: ${value}
`; } }