Merge pull request #5424 from HofiOne/add-without-orphaned-option-to-stats

HofiOne · web-flow · commit 9e97f81bd892 · 2025-07-25T12:56:05.000+02:00
Add `without-orphaned-metrics` option to stats
diff --git a/lib/logproto/logproto-http-scraper-responder-server.c b/lib/logproto/logproto-http-scraper-responder-server.c
@@ -72,16 +72,18 @@ _compose_response_body(LogProtoHTTPServer *s)
     {
       stats = g_string_new(NULL);
       gpointer args[] = {self, &stats};
-      gboolean with_legacy = TRUE;
+      gboolean with_legacy = self->options->stats_with_legacy;
+      gboolean without_orphaned = self->options->stats_without_orphaned;
 
       msg_trace("Generating stats", evt_tag_str("stat-format", stat_format),
-                evt_tag_int("with-legacy", with_legacy));
+                evt_tag_int("with-legacy", with_legacy),
+                evt_tag_int("without-orphaned", without_orphaned));
       if (strcmp(stat_format, "prometheus") == 0)
-        stats_generate_prometheus(_generate_batched_response, args, with_legacy, &cancelled);
+        stats_generate_prometheus(_generate_batched_response, args, with_legacy, without_orphaned, &cancelled);
       else
         {
           gboolean csv = (strcmp(stat_format, "csv") == 0);
-          stats_generate_csv_or_kv(_generate_batched_response, args, csv, FALSE, &cancelled);
+          stats_generate_csv_or_kv(_generate_batched_response, args, csv, FALSE, without_orphaned, &cancelled);
         }
     }
   else
@@ -329,3 +331,21 @@ log_proto_http_scraper_responder_options_set_stat_format(LogProtoServerOptionsSt
     }
   return FALSE;
 }
+
+void
+log_proto_http_scraper_responder_options_set_stats_without_orpaned(LogProtoServerOptionsStorage *options_storage,
+    gboolean value)
+{
+  LogProtoHTTPScraperResponderOptions *options = &((LogProtoHTTPScraperResponderOptionsStorage *)options_storage)->super;
+
+  options->stats_without_orphaned = value;
+}
+
+void
+log_proto_http_scraper_responder_options_set_stats_with_legacy(LogProtoServerOptionsStorage *options_storage,
+    gboolean value)
+{
+  LogProtoHTTPScraperResponderOptions *options = &((LogProtoHTTPScraperResponderOptionsStorage *)options_storage)->super;
+
+  options->stats_with_legacy = value;
+}
diff --git a/lib/logproto/logproto-http-scraper-responder-server.h b/lib/logproto/logproto-http-scraper-responder-server.h
@@ -40,6 +40,8 @@ typedef struct _LogProtoHTTPScraperResponderOptions
   gchar *stat_query;
   gchar *stat_format;
   gboolean single_instance;
+  gboolean stats_without_orphaned;
+  gboolean stats_with_legacy;
 
 } LogProtoHTTPScraperResponderOptions;
 
@@ -85,6 +87,12 @@ void log_proto_http_scraper_responder_options_set_stat_query(
 gboolean log_proto_http_scraper_responder_options_set_stat_format(
   LogProtoServerOptionsStorage *options,
   const gchar *value);
+void log_proto_http_scraper_responder_options_set_stats_without_orpaned(
+  LogProtoServerOptionsStorage *options_storage,
+  gboolean value);
+void log_proto_http_scraper_responder_options_set_stats_with_legacy(
+  LogProtoServerOptionsStorage *options_storage,
+  gboolean value);
 
 LogProtoServer *log_proto_http_scraper_responder_server_new(
   LogTransport *transport,
diff --git a/lib/logproto/tests/test-http-scraper-responder-server.c b/lib/logproto/tests/test-http-scraper-responder-server.c
@@ -136,7 +136,7 @@ test_empty_crnl_multiline_at_eof(LogTransportMockConstructor log_transport_mock_
 // Mock of LogProtoHTTPScraperResponder::stats_generate_prometheus
 void
 stats_generate_prometheus(StatsPrometheusRecordFunc process_record, gpointer user_data, gboolean with_legacy,
-                          gboolean *cancelled)
+                          gboolean without_orphaned, gboolean *cancelled)
 {
   gpointer *args = (gpointer *)user_data;
   GString *result = *(GString **)args[1];
diff --git a/lib/stats/stats-control.c b/lib/stats/stats-control.c
@@ -78,23 +78,39 @@ static void
 control_connection_send_stats(ControlConnection *cc, GString *command, gpointer user_data, gboolean *cancelled)
 {
   gchar **cmds = g_strsplit(command->str, " ", 3);
-  g_assert(g_str_equal(cmds[0], "STATS"));
-  g_assert(cmds[1] != NULL && "STATS command must have at least one format argument");
+  gsize cmd_ndx = 0;
+  g_assert(g_str_equal(cmds[cmd_ndx], "STATS"));
+  ++cmd_ndx;
+  g_assert(cmds[cmd_ndx] != NULL && "STATS command must have at least one format argument");
 
   GString *response = NULL;
   gpointer args[] = {cc, &response};
 
-  if (strcmp(cmds[1], "PROMETHEUS") == 0)
+  if (strcmp(cmds[cmd_ndx], "PROMETHEUS") == 0)
     {
-      gboolean with_legacy = g_strcmp0(cmds[2], "WITH_LEGACY") == 0;
-      stats_generate_prometheus(_send_batched_response, args, with_legacy, cancelled);
+      gboolean with_legacy = g_strcmp0(cmds[cmd_ndx + 1], "WITH_LEGACY") == 0;
+      if (with_legacy)
+        ++cmd_ndx;
+      gboolean without_orphaned = g_strcmp0(cmds[cmd_ndx + 1], "WITHOUT_ORPHANED") == 0;
+      // NOTE: do not forget to increment cmd_ndx if new commands added later
+      //       now just commented out to avoid compiler warning
+      // if (without_orphaned)
+      //   ++cmd_ndx;
+      stats_generate_prometheus(_send_batched_response, args, with_legacy, without_orphaned, cancelled);
     }
   else
     {
-      gboolean csv = strcmp(cmds[1], "CSV") == 0;
-      g_assert(csv || strcmp(cmds[1], "KV") == 0);
-      gboolean without_header = g_strcmp0(cmds[2], "WITHOUT_HEADER") == 0;
-      stats_generate_csv_or_kv(_send_batched_response, args, csv, FALSE == without_header, cancelled);
+      gboolean csv = strcmp(cmds[cmd_ndx], "CSV") == 0;
+      g_assert(csv || strcmp(cmds[cmd_ndx], "KV") == 0);
+      gboolean without_header = g_strcmp0(cmds[cmd_ndx + 1], "WITHOUT_HEADER") == 0;
+      if (without_header)
+        ++cmd_ndx;
+      gboolean without_orphaned = g_strcmp0(cmds[cmd_ndx + 1], "WITHOUT_ORPHANED") == 0;
+      // NOTE: do not forget to increment cmd_ndx if new commands added later
+      //       now just commented out to avoid compiler warning
+      // if (without_orphaned)
+      //   ++cmd_ndx;
+      stats_generate_csv_or_kv(_send_batched_response, args, csv, FALSE == without_header, without_orphaned, cancelled);
     }
 
   if (response != NULL)
diff --git a/lib/stats/stats-csv.c b/lib/stats/stats-csv.c
@@ -97,6 +97,10 @@ stats_format_csv_or_kv(StatsCluster *sc, gint type, StatsCounterItem *counter, g
   StatsCSVRecordFunc process_record = (StatsCSVRecordFunc) args[0];
   gpointer process_record_arg = args[1];
   gboolean csv = GPOINTER_TO_INT(args[2]);
+  gboolean without_orphaned = GPOINTER_TO_INT(args[3]);
+
+  if (without_orphaned && stats_cluster_is_orphaned(sc))
+    return;
 
   ScratchBuffersMarker marker;
   scratch_buffers_mark(&marker);
@@ -119,7 +123,7 @@ stats_format_csv_or_kv(StatsCluster *sc, gint type, StatsCounterItem *counter, g
 
 void
 stats_generate_csv_or_kv(StatsCSVRecordFunc process_record, gpointer user_data, gboolean csv, gboolean with_header,
-                         gboolean *cancelled)
+                         gboolean without_orphaned, gboolean *cancelled)
 {
   if (with_header && csv)
     {
@@ -130,7 +134,7 @@ stats_generate_csv_or_kv(StatsCSVRecordFunc process_record, gpointer user_data,
       process_record(header->str, user_data);
       g_string_free(header, TRUE);
     }
-  gpointer format_args[] = {process_record, user_data, GINT_TO_POINTER(csv)};
+  gpointer format_args[] = {process_record, user_data, GINT_TO_POINTER(csv), GINT_TO_POINTER(without_orphaned)};
   stats_lock();
   stats_foreach_legacy_counter(stats_format_csv_or_kv, format_args, cancelled);
   stats_unlock();
diff --git a/lib/stats/stats-csv.h b/lib/stats/stats-csv.h
@@ -32,6 +32,6 @@ typedef void (*StatsCSVRecordFunc)(const char *record, gpointer user_data);
 GString *stats_csv_format_counter(StatsCluster *sc, gint type, StatsCounterItem *counter);
 
 void stats_generate_csv_or_kv(StatsCSVRecordFunc process_record, gpointer user_data, gboolean csv,
-                              gboolean with_header, gboolean *cancelled);
+                              gboolean with_header, gboolean without_orphaned, gboolean *cancelled);
 
 #endif
diff --git a/lib/stats/stats-prometheus.c b/lib/stats/stats-prometheus.c
@@ -294,11 +294,12 @@ stats_format_prometheus(StatsCluster *sc, gint type, StatsCounterItem *counter,
   StatsPrometheusRecordFunc process_record = (StatsPrometheusRecordFunc) args[0];
   gpointer process_record_arg = args[1];
   gboolean with_legacy = GPOINTER_TO_INT(args[2]);
+  gboolean without_orphaned = GPOINTER_TO_INT(args[3]);
 
   if (!sc->key.name && !with_legacy)
     return;
 
-  if (stats_cluster_is_orphaned(sc))
+  if (without_orphaned && stats_cluster_is_orphaned(sc))
     return;
 
   ScratchBuffersMarker marker;
@@ -314,9 +315,9 @@ stats_format_prometheus(StatsCluster *sc, gint type, StatsCounterItem *counter,
 
 void
 stats_generate_prometheus(StatsPrometheusRecordFunc process_record, gpointer user_data, gboolean with_legacy,
-                          gboolean *cancelled)
+                          gboolean without_orphaned, gboolean *cancelled)
 {
-  gpointer format_prometheus_args[] = {process_record, user_data, GINT_TO_POINTER(with_legacy)};
+  gpointer format_prometheus_args[] = {process_record, user_data, GINT_TO_POINTER(with_legacy), GINT_TO_POINTER(without_orphaned)};
   stats_lock();
   stats_foreach_counter(stats_format_prometheus, format_prometheus_args, cancelled);
   stats_unlock();
diff --git a/lib/stats/stats-prometheus.h b/lib/stats/stats-prometheus.h
@@ -33,6 +33,6 @@ typedef void (*StatsPrometheusRecordFunc)(const char *record, gpointer user_data
 GString *stats_prometheus_format_counter(StatsCluster *sc, gint type, StatsCounterItem *counter);
 
 void stats_generate_prometheus(StatsPrometheusRecordFunc process_record, gpointer user_data, gboolean with_legacy,
-                               gboolean *cancelled);
+                               gboolean without_orphaned, gboolean *cancelled);
 
 #endif
diff --git a/modules/afsocket/afsocket-grammar.ym b/modules/afsocket/afsocket-grammar.ym
@@ -152,6 +152,8 @@ systemd_syslog_grammar_set_source_driver(SystemDSyslogSourceDriver *sd)
 %token KW_STAT_TYPE
 %token KW_STAT_QUERY
 %token KW_STAT_FORMAT
+%token KW_STATS_WITHOUT_ORPHANED
+%token KW_STATS_WITH_LEGACY
 
 %token KW_IP_TTL
 %token KW_SO_BROADCAST
@@ -490,6 +492,8 @@ stats_option
 	: KW_STAT_TYPE '(' string ')' { CHECK_ERROR(log_proto_http_scraper_responder_options_set_stat_type(last_proto_server_options, $3), @3, "Invalid prometheus scraper stat type: %s", $3); free($3); }
 	| KW_STAT_QUERY '(' string ')' { log_proto_http_scraper_responder_options_set_stat_query(last_proto_server_options, $3); free($3); }
 	| KW_STAT_FORMAT '(' string ')' { CHECK_ERROR(log_proto_http_scraper_responder_options_set_stat_format(last_proto_server_options, $3), @3, "Invalid prometheus scraper stat format: %s", $3); free($3); }
+	| KW_STATS_WITHOUT_ORPHANED '(' yesno ')' { log_proto_http_scraper_responder_options_set_stats_without_orpaned(last_proto_server_options, $3); }
+	| KW_STATS_WITH_LEGACY '(' yesno ')' { log_proto_http_scraper_responder_options_set_stats_with_legacy(last_proto_server_options, $3); }
 	;
 
 stats_options
diff --git a/modules/afsocket/afsocket-parser.c b/modules/afsocket/afsocket-parser.c
@@ -32,8 +32,8 @@ int afsocket_parse(CfgLexer *lexer, LogDriver **instance, gpointer arg);
 
 static CfgLexerKeyword afsocket_keywords[] =
 {
-  { "unix_dgram", KW_UNIX_DGRAM },
-  { "unix_stream",  KW_UNIX_STREAM },
+  { "unix_dgram",         KW_UNIX_DGRAM },
+  { "unix_stream",        KW_UNIX_STREAM },
   { "udp",                KW_UDP },
   { "tcp",                KW_TCP },
   { "syslog",             KW_SYSLOG },
@@ -47,6 +47,8 @@ static CfgLexerKeyword afsocket_keywords[] =
   { "stat_type",          KW_STAT_TYPE},
   { "stat_query",         KW_STAT_QUERY},
   { "stat_format",        KW_STAT_FORMAT},
+  { "stats_without_orphaned", KW_STATS_WITHOUT_ORPHANED },
+  { "stats_with_legacy",  KW_STATS_WITH_LEGACY },
 
 #if SYSLOG_NG_ENABLE_IPV6
   { "udp6",               KW_UDP6 },
diff --git a/scl/stats-exporter/plugin.conf b/scl/stats-exporter/plugin.conf
@@ -28,6 +28,8 @@ block source stats-exporter(
     stat-type("stats")
     stat-query("*")
     stat-format("prometheus")
+    stats-without-orphaned(no)
+    stats-with-legacy(no)
     ip()
     port()
     flags(no-parse dont-store-legacy-msghdr)
@@ -50,6 +52,8 @@ block source stats-exporter(
                         stat-type(`stat-type`)
                         stat-query(`stat-query`)
                         stat-format(`stat-format`)
+                        stats-without-orphaned(`stats-without-orphaned`)
+                        stats-with-legacy(`stats-with-legacy`)
                     )
                 )
             );
diff --git a/syslog-ng-ctl/commands/ctl-stats.c b/syslog-ng-ctl/commands/ctl-stats.c
@@ -27,18 +27,19 @@
 static gboolean stats_options_reset_is_set = FALSE;
 static gboolean stats_options_remove_orphans = FALSE;
 static gboolean stats_options_legacy_metrics = FALSE;
+static gboolean stats_options_without_orphaned = FALSE;
 static gboolean stats_options_without_header = FALSE;
 static gchar *stats_options_out_format = NULL;
 static gchar **stats_legacy_commands = NULL;
 
-static const gchar *
+static GString *
 _stats_command_builder(GOptionContext *ctx)
 {
   if (stats_options_reset_is_set)
-    return "RESET_STATS";
+    return g_string_new("RESET_STATS");
 
   if (stats_options_remove_orphans)
-    return "REMOVE_ORPHANED_STATS";
+    return g_string_new("REMOVE_ORPHANED_STATS");
 
   /* NOTE: Yes, this is a mess, but it's a legacy mess, so we have to keep it currently to be backward compatible.
    *       Unfortunately, there's no optional command construct in the GLib Commandline Option Parser
@@ -75,33 +76,37 @@ _stats_command_builder(GOptionContext *ctx)
       return NULL;
     }
 
+  GString *stats_format = g_string_sized_new(64);
+  g_string_append(stats_format, "STATS");
   if ((stats_command == NULL && stats_options_out_format == NULL) ||
       command_is_csv || option_is_csv)
     {
       if (option_is_prometheus || command_is_prometheus || option_is_kv || command_is_kv)
         goto on_collision;
+      g_string_append(stats_format, " CSV");
       if (stats_options_without_header)
-        return "STATS CSV WITHOUT_HEADER";
-      else
-        return "STATS CSV";
+        g_string_append(stats_format, " WITHOUT_HEADER");
     }
   else if (command_is_kv || option_is_kv)
     {
       if (option_is_csv || command_is_csv || option_is_prometheus || command_is_prometheus)
         goto on_collision;
-      return "STATS KV";
+      g_string_append(stats_format, " KV");
     }
   else if (command_is_prometheus || option_is_prometheus)
     {
       if (option_is_csv || command_is_csv || option_is_kv || command_is_kv)
         goto on_collision;
+      g_string_append(stats_format, " PROMETHEUS");
       if (stats_options_legacy_metrics)
-        return "STATS PROMETHEUS WITH_LEGACY";
-      else
-        return "STATS PROMETHEUS";
+        g_string_append(stats_format, " WITH_LEGACY");
     }
+  if (stats_options_without_orphaned)
+    g_string_append(stats_format, " WITHOUT_ORPHANED");
+  return stats_format;
 
 on_collision:
+  g_string_free(stats_format, TRUE);
   g_option_context_set_description(ctx, "stats");
   fprintf(stderr,
           "Error parsing command line arguments: Colliding stats format recieved: %s <-> %s\nUsing the `stats csv|kv|stats prometheus` format is deprecated, please use the new `--%s csv|--%s kv|--%s prometheus` options instead\n\n",
@@ -113,18 +118,21 @@ _stats_command_builder(GOptionContext *ctx)
 gint
 slng_stats(int argc, char *argv[], const gchar *mode, GOptionContext *ctx)
 {
-  const gchar *command = _stats_command_builder(ctx);
+  GString *command = _stats_command_builder(ctx);
   if (command == NULL)
     return EINVAL;
 
-  return dispatch_command(command);
+  gint result = dispatch_command(command->str);
+  g_string_free(command, TRUE);
+  return result;
 }
 
 GOptionEntry stats_options[] =
 {
   { "reset", 'r', 0, G_OPTION_ARG_NONE, &stats_options_reset_is_set, "reset counters", NULL },
   { "remove-orphans", 'o', 0, G_OPTION_ARG_NONE, &stats_options_remove_orphans, "remove orphaned statistics", NULL},
   { "with-legacy-metrics", 'l', 0, G_OPTION_ARG_NONE, &stats_options_legacy_metrics, "show legacy metrics (only for prometheus format)", NULL},
+  { "without-orphaned-metrics", 'a', 0, G_OPTION_ARG_NONE, &stats_options_without_orphaned, "do not show orphaned metrics", NULL},
   { "no-header", 'n', 0, G_OPTION_ARG_NONE, &stats_options_without_header, "do not print stats header", NULL},
   { "format", 'm', 0, G_OPTION_ARG_STRING, &stats_options_out_format, "output format, default is <csv>", "<csv|kv|prometheus>" },
   { G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_STRING_ARRAY, &stats_legacy_commands, NULL, NULL },

Original file line number	Diff line number	Diff line change
`@@ -136,7 +136,7 @@ test_empty_crnl_multiline_at_eof(LogTransportMockConstructor log_transport_mock_`
`136`	`136`	`// Mock of LogProtoHTTPScraperResponder::stats_generate_prometheus`
`137`	`137`	`void`
`138`	`138`	`stats_generate_prometheus(StatsPrometheusRecordFunc process_record, gpointer user_data, gboolean with_legacy,`
`139`		`- gboolean *cancelled)`
	`139`	`+ gboolean without_orphaned, gboolean *cancelled)`
`140`	`140`	`{`
`141`	`141`	`gpointer args = (gpointer )user_data;`
`142`	`142`	`GString result = (GString **)args[1];`