chore(observability): set source fields to mean service (vectordotdev…

…#17470) This adds the `service` meaning to the `appname` field from syslog, `app_name` from `heroku_logs`, `source` from `splunk_hec`. It also adds a new field to `demo_logs` called `service` and populates it with the value `vector`. The `datadog_agent` source already [handles this](https://github.com/vectordotdev/vector/blob/master/src/sources/datadog_agent/mod.rs#L225-L231). I can't think of any other sources that may specify a potential service field, but I am very open to suggestions! --------- Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com>
aholmberg · May 25, 2023 · 670bdea · 670bdea
1 parent 426d660
commit 670bdea
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 15 deletions.
diff --git a/lib/codecs/src/decoding/format/syslog.rs b/lib/codecs/src/decoding/format/syslog.rs
@@ -71,7 +71,11 @@ impl SyslogDeserializerConfig {
                     )
                     .optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
                     .optional_field(&owned_value_path!("version"), Kind::integer(), None)
-                    .optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
+                    .optional_field(
+                        &owned_value_path!("appname"),
+                        Kind::bytes(),
+                        Some("service"),
+                    )
                     .optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
                     .optional_field(
                         &owned_value_path!("procid"),
@@ -112,7 +116,11 @@ impl SyslogDeserializerConfig {
                 )
                 .optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
                 .optional_field(&owned_value_path!("version"), Kind::integer(), None)
-                .optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
+                .optional_field(
+                    &owned_value_path!("appname"),
+                    Kind::bytes(),
+                    Some("service"),
+                )
                 .optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
                 .optional_field(
                     &owned_value_path!("procid"),
@@ -172,7 +180,7 @@ impl SyslogDeserializerConfig {
                         None,
                         &owned_value_path!("appname"),
                         Kind::bytes().or_undefined(),
-                        None,
+                        Some("service"),
                     )
                     .with_source_metadata(
                         source,

diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs
@@ -1690,7 +1690,11 @@ fn test_config_outputs() {
                             )
                             .optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
                             .optional_field(&owned_value_path!("version"), Kind::integer(), None)
-                            .optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
+                            .optional_field(
+                                &owned_value_path!("appname"),
+                                Kind::bytes(),
+                                Some("service"),
+                            )
                             .optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
                             .optional_field(
                                 &owned_value_path!("procid"),
@@ -1766,7 +1770,11 @@ fn test_config_outputs() {
                                     Kind::integer(),
                                     None,
                                 )
-                                .optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
+                                .optional_field(
+                                    &owned_value_path!("appname"),
+                                    Kind::bytes(),
+                                    Some("service"),
+                                )
                                 .optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
                                 .optional_field(
                                     &owned_value_path!("procid"),

diff --git a/src/sources/demo_logs.rs b/src/sources/demo_logs.rs
@@ -5,6 +5,7 @@ use codecs::{
 };
 use fakedata::logs::*;
 use futures::StreamExt;
+use lookup::{owned_value_path, path};
 use rand::seq::SliceRandom;
 use serde_with::serde_as;
 use snafu::Snafu;
@@ -15,7 +16,11 @@ use vector_common::internal_event::{
     ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol,
 };
 use vector_config::configurable_component;
-use vector_core::{config::LogNamespace, EstimatedJsonEncodedSizeOf};
+use vector_core::{
+    config::{LegacyKey, LogNamespace},
+    EstimatedJsonEncodedSizeOf,
+};
+use vrl::value::Kind;
 
 use crate::{
     codecs::{Decoder, DecodingConfig},
@@ -249,6 +254,13 @@ async fn demo_logs_source(
                             DemoLogsConfig::NAME,
                             now,
                         );
+                        log_namespace.insert_source_metadata(
+                            "service",
+                            log,
+                            Some(LegacyKey::InsertIfEmpty(path!("service"))),
+                            path!("service"),
+                            "vector",
+                        );
 
                         event
                     });
@@ -300,7 +312,14 @@ impl SourceConfig for DemoLogsConfig {
         let schema_definition = self
             .decoding
             .schema_definition(log_namespace)
-            .with_standard_vector_source_metadata();
+            .with_standard_vector_source_metadata()
+            .with_source_metadata(
+                DemoLogsConfig::NAME,
+                Some(LegacyKey::InsertIfEmpty(owned_value_path!("service"))),
+                &owned_value_path!("service"),
+                Kind::bytes(),
+                Some("service"),
+            );
 
         vec![SourceOutput::new_logs(
             self.decoding.output_type(),

diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs
@@ -107,7 +107,7 @@ impl LogplexConfig {
                 Some(LegacyKey::InsertIfEmpty(owned_value_path!("app_name"))),
                 &owned_value_path!("app_name"),
                 Kind::bytes(),
-                None,
+                Some("service"),
             )
             .with_source_metadata(
                 LogplexConfig::NAME,
@@ -694,7 +694,7 @@ mod tests {
                 .with_metadata_field(
                     &owned_value_path!(LogplexConfig::NAME, "app_name"),
                     Kind::bytes(),
-                    None,
+                    Some("service"),
                 )
                 .with_metadata_field(
                     &owned_value_path!(LogplexConfig::NAME, "proc_id"),
@@ -731,7 +731,11 @@ mod tests {
         .with_event_field(&owned_value_path!("source_type"), Kind::bytes(), None)
         .with_event_field(&owned_value_path!("timestamp"), Kind::timestamp(), None)
         .with_event_field(&owned_value_path!("host"), Kind::bytes(), Some("host"))
-        .with_event_field(&owned_value_path!("app_name"), Kind::bytes(), None)
+        .with_event_field(
+            &owned_value_path!("app_name"),
+            Kind::bytes(),
+            Some("service"),
+        )
         .with_event_field(&owned_value_path!("proc_id"), Kind::bytes(), None)
         .unknown_fields(Kind::bytes());
 

diff --git a/src/sources/splunk_hec/mod.rs b/src/sources/splunk_hec/mod.rs
@@ -226,7 +226,7 @@ impl SourceConfig for SplunkConfig {
             Some(LegacyKey::Overwrite(owned_value_path!(SOURCE))),
             &owned_value_path!("source"),
             Kind::bytes(),
-            None,
+            Some("service"),
         )
         // Not to be confused with `source_type`.
         .with_source_metadata(
@@ -2475,7 +2475,7 @@ mod tests {
         .with_metadata_field(
             &owned_value_path!("splunk_hec", "source"),
             Kind::bytes(),
-            None,
+            Some("service"),
         )
         .with_metadata_field(
             &owned_value_path!("splunk_hec", "channel"),
@@ -2519,7 +2519,11 @@ mod tests {
         .with_event_field(&owned_value_path!("source_type"), Kind::bytes(), None)
         .with_event_field(&owned_value_path!("splunk_channel"), Kind::bytes(), None)
         .with_event_field(&owned_value_path!("splunk_index"), Kind::bytes(), None)
-        .with_event_field(&owned_value_path!("splunk_source"), Kind::bytes(), None)
+        .with_event_field(
+            &owned_value_path!("splunk_source"),
+            Kind::bytes(),
+            Some("service"),
+        )
         .with_event_field(&owned_value_path!("splunk_sourcetype"), Kind::bytes(), None)
         .with_event_field(&owned_value_path!("timestamp"), Kind::timestamp(), None);
 

diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs
@@ -549,7 +549,7 @@ mod test {
                 .with_metadata_field(
                     &owned_value_path!("syslog", "appname"),
                     Kind::bytes().or_undefined(),
-                    None,
+                    Some("service"),
                 )
                 .with_metadata_field(
                     &owned_value_path!("syslog", "msgid"),
@@ -628,7 +628,7 @@ mod test {
         .with_event_field(
             &owned_value_path!("appname"),
             Kind::bytes().or_undefined(),
-            None,
+            Some("service"),
         )
         .with_event_field(
             &owned_value_path!("msgid"),