From 6057dc7cb2f09ac097b17b4c9674cac72be4206c Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Fri, 26 Apr 2024 12:53:48 +0530 Subject: [PATCH 1/8] feat(new encoding): add pretty json encoding --- lib/codecs/src/encoding/format/mod.rs | 2 + lib/codecs/src/encoding/format/pretty_json.rs | 239 ++++++++++++++++++ lib/codecs/src/encoding/mod.rs | 17 +- src/codecs/encoding/config.rs | 2 +- src/codecs/encoding/encoder.rs | 5 +- src/components/validation/resources/mod.rs | 1 + src/sinks/websocket/sink.rs | 4 +- 7 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 lib/codecs/src/encoding/format/pretty_json.rs diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index e61f7cae0bb96..d45910bd8651a 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -14,6 +14,7 @@ mod native_json; mod protobuf; mod raw_message; mod text; +mod pretty_json; use std::fmt::Debug; @@ -22,6 +23,7 @@ pub use avro::{AvroSerializer, AvroSerializerConfig, AvroSerializerOptions}; use dyn_clone::DynClone; pub use gelf::{GelfSerializer, GelfSerializerConfig}; pub use json::{JsonSerializer, JsonSerializerConfig}; +pub use pretty_json::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; pub use logfmt::{LogfmtSerializer, LogfmtSerializerConfig}; pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; diff --git a/lib/codecs/src/encoding/format/pretty_json.rs b/lib/codecs/src/encoding/format/pretty_json.rs new file mode 100644 index 0000000000000..6f56742fb6fce --- /dev/null +++ b/lib/codecs/src/encoding/format/pretty_json.rs @@ -0,0 +1,239 @@ +use bytes::{BufMut, BytesMut}; +use tokio_util::codec::Encoder; +use vector_core::{config::DataType, event::Event, schema}; + +use crate::MetricTagValues; + +/// Config used to build a `JsonSerializer`. +#[crate::configurable_component] +#[derive(Debug, Clone, Default)] +pub struct PrettyJsonSerializerConfig { + /// Controls how metric tag values are encoded. + /// + /// When set to `single`, only the last non-bare value of tags are displayed with the + /// metric. When set to `full`, all metric tags are exposed as separate assignments. + #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] + pub metric_tag_values: MetricTagValues, +} + +impl PrettyJsonSerializerConfig { + /// Creates a new `JsonSerializerConfig`. + pub const fn new(metric_tag_values: MetricTagValues) -> Self { + Self { metric_tag_values } + } + + /// Build the `JsonSerializer` from this configuration. + pub const fn build(&self) -> PrettyJsonSerializer { + PrettyJsonSerializer::new(self.metric_tag_values) + } + + /// The data type of events that are accepted by `JsonSerializer`. + pub fn input_type(&self) -> DataType { + DataType::all() + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + // While technically we support `Value` variants that can't be losslessly serialized to + // PrettyJson, we don't want to enforce that limitation to users yet. + schema::Requirement::empty() + } +} + +/// Serializer that converts an `Event` to bytes using the PrettyJson format. +#[derive(Debug, Clone)] +pub struct PrettyJsonSerializer { + metric_tag_values: MetricTagValues, +} + +impl PrettyJsonSerializer { + /// Creates a new `JsonSerializer`. + pub const fn new(metric_tag_values: MetricTagValues) -> Self { + Self { metric_tag_values } + } + + /// Encode event and represent it as PrettyJson value. + pub fn to_json_value(&self, event: Event) -> Result { + match event { + Event::Log(log) => serde_json::to_value(&log), + Event::Metric(metric) => serde_json::to_value(&metric), + Event::Trace(trace) => serde_json::to_value(&trace), + } + .map_err(|e| e.to_string().into()) + } +} + +impl Encoder for PrettyJsonSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + let writer = buffer.writer(); + match event { + Event::Log(log) => serde_json::to_writer_pretty(writer, &log), + Event::Metric(mut metric) => { + if self.metric_tag_values == MetricTagValues::Single { + metric.reduce_tags_to_single(); + } + serde_json::to_writer_pretty(writer, &metric) + } + Event::Trace(trace) => serde_json::to_writer_pretty(writer, &trace), + } + .map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use bytes::{Bytes, BytesMut}; + use chrono::{TimeZone, Timelike, Utc}; + use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, StatisticKind, Value}; + use vector_core::metric_tags; + use vrl::btreemap; + + use super::*; + + #[test] + fn serialize_json_log() { + let event = Event::Log(LogEvent::from(btreemap! { + "x" => Value::from("23"), + "z" => Value::from(25), + "a" => Value::from("0"), + })); + let bytes = serialize(JsonSerializerConfig::default(), event); + + assert_eq!(bytes, r#"{"a":"0","x":"23","z":25}"#); + } + + #[test] + fn serialize_json_metric_counter() { + let event = Event::Metric( + Metric::new( + "foos", + MetricKind::Incremental, + MetricValue::Counter { value: 100.0 }, + ) + .with_namespace(Some("vector")) + .with_tags(Some(metric_tags!( + "key2" => "value2", + "key1" => "value1", + "Key3" => "Value3", + ))) + .with_timestamp(Some( + Utc.with_ymd_and_hms(2018, 11, 14, 8, 9, 10) + .single() + .and_then(|t| t.with_nanosecond(11)) + .expect("invalid timestamp"), + )), + ); + + let bytes = serialize(JsonSerializerConfig::default(), event); + + assert_eq!( + bytes, + r#"{"name":"foos","namespace":"vector","tags":{"Key3":"Value3","key1":"value1","key2":"value2"},"timestamp":"2018-11-14T08:09:10.000000011Z","kind":"incremental","counter":{"value":100.0}}"# + ); + } + + #[test] + fn serialize_json_metric_set() { + let event = Event::Metric(Metric::new( + "users", + MetricKind::Incremental, + MetricValue::Set { + values: vec!["bob".into()].into_iter().collect(), + }, + )); + + let bytes = serialize(JsonSerializerConfig::default(), event); + + assert_eq!( + bytes, + r#"{"name":"users","kind":"incremental","set":{"values":["bob"]}}"# + ); + } + + #[test] + fn serialize_json_metric_histogram_without_timestamp() { + let event = Event::Metric(Metric::new( + "glork", + MetricKind::Incremental, + MetricValue::Distribution { + samples: vector_core::samples![10.0 => 1], + statistic: StatisticKind::Histogram, + }, + )); + + let bytes = serialize(JsonSerializerConfig::default(), event); + + assert_eq!( + bytes, + r#"{"name":"glork","kind":"incremental","distribution":{"samples":[{"value":10.0,"rate":1}],"statistic":"histogram"}}"# + ); + } + + #[test] + fn serialize_equals_to_json_value() { + let event = Event::Log(LogEvent::from(btreemap! { + "foo" => Value::from("bar") + })); + let mut serializer = PrettyJsonSerializerConfig::default().build(); + let mut bytes = BytesMut::new(); + + serializer.encode(event.clone(), &mut bytes).unwrap(); + + let json = serializer.to_json_value(event).unwrap(); + + assert_eq!(bytes.freeze(), serde_json::to_string_pretty(&json).unwrap()); + } + + #[test] + fn serialize_metric_tags_full() { + let bytes = serialize( + PrettyJsonSerializerConfig { + metric_tag_values: MetricTagValues::Full, + }, + metric2(), + ); + + assert_eq!( + bytes, + r#"{"name":"counter","tags":{"a":["first",null,"second"]},"kind":"incremental","counter":{"value":1.0}}"# + ); + } + + #[test] + fn serialize_metric_tags_single() { + let bytes = serialize( + PrettyJsonSerializerConfig { + metric_tag_values: MetricTagValues::Single, + }, + metric2(), + ); + + assert_eq!( + bytes, + r#"{"name":"counter","tags":{"a":"second"},"kind":"incremental","counter":{"value":1.0}}"# + ); + } + + fn metric2() -> Event { + Event::Metric( + Metric::new( + "counter", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + ) + .with_tags(Some(metric_tags! ( + "a" => "first", + "a" => None, + "a" => "second", + ))), + ) + } + + fn serialize(config: PrettyJsonSerializerConfig, input: Event) -> Bytes { + let mut buffer = BytesMut::new(); + config.build().encode(input, &mut buffer).unwrap(); + buffer.freeze() + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 0d766f73e4d17..36398e54fe314 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -23,6 +23,8 @@ pub use framing::{ use vector_config::configurable_component; use vector_core::{config::DataType, event::Event, schema}; +use self::format::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; + /// An error that occurred while building an encoder. pub type BuildError = Box; @@ -213,6 +215,11 @@ pub enum SerializerConfig { /// [json]: https://www.json.org/ Json(JsonSerializerConfig), + /// Encodes an event as [JSON][json] in pretty format. + /// + /// [json]: https://www.json.org/ + PrettyJson(PrettyJsonSerializerConfig), + /// Encodes an event as a [logfmt][logfmt] message. /// /// [logfmt]: https://brandur.org/logfmt @@ -329,6 +336,7 @@ impl SerializerConfig { SerializerConfig::Csv(config) => Ok(Serializer::Csv(config.build()?)), SerializerConfig::Gelf => Ok(Serializer::Gelf(GelfSerializerConfig::new().build())), SerializerConfig::Json(config) => Ok(Serializer::Json(config.build())), + SerializerConfig::PrettyJson(config) => Ok(Serializer::PrettyJson(config.build())), SerializerConfig::Logfmt => Ok(Serializer::Logfmt(LogfmtSerializerConfig.build())), SerializerConfig::Native => Ok(Serializer::Native(NativeSerializerConfig.build())), SerializerConfig::NativeJson => { @@ -364,6 +372,7 @@ impl SerializerConfig { SerializerConfig::Csv(_) | SerializerConfig::Gelf | SerializerConfig::Json(_) + | SerializerConfig::PrettyJson(_) | SerializerConfig::Logfmt | SerializerConfig::NativeJson | SerializerConfig::RawMessage @@ -380,6 +389,7 @@ impl SerializerConfig { SerializerConfig::Csv(config) => config.input_type(), SerializerConfig::Gelf { .. } => GelfSerializerConfig::input_type(), SerializerConfig::Json(config) => config.input_type(), + SerializerConfig::PrettyJson(config) => config.input_type(), SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.input_type(), @@ -398,6 +408,7 @@ impl SerializerConfig { SerializerConfig::Csv(config) => config.schema_requirement(), SerializerConfig::Gelf { .. } => GelfSerializerConfig::schema_requirement(), SerializerConfig::Json(config) => config.schema_requirement(), + SerializerConfig::PrettyJson(config) => config.schema_requirement(), SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.schema_requirement(), @@ -419,6 +430,8 @@ pub enum Serializer { Gelf(GelfSerializer), /// Uses a `JsonSerializer` for serialization. Json(JsonSerializer), + /// Uses a `JsonSerializer` for serialization. + PrettyJson(PrettyJsonSerializer), /// Uses a `LogfmtSerializer` for serialization. Logfmt(LogfmtSerializer), /// Uses a `NativeSerializer` for serialization. @@ -437,7 +450,7 @@ impl Serializer { /// Check if the serializer supports encoding an event to JSON via `Serializer::to_json_value`. pub fn supports_json(&self) -> bool { match self { - Serializer::Json(_) | Serializer::NativeJson(_) | Serializer::Gelf(_) => true, + Serializer::Json(_) | Self::PrettyJson(_) | Serializer::NativeJson(_) | Serializer::Gelf(_) => true, Serializer::Avro(_) | Serializer::Csv(_) | Serializer::Logfmt(_) @@ -458,6 +471,7 @@ impl Serializer { match self { Serializer::Gelf(serializer) => serializer.to_json_value(event), Serializer::Json(serializer) => serializer.to_json_value(event), + Serializer::PrettyJson(serializer) => serializer.to_json_value(event), Serializer::NativeJson(serializer) => serializer.to_json_value(event), Serializer::Avro(_) | Serializer::Csv(_) @@ -541,6 +555,7 @@ impl tokio_util::codec::Encoder for Serializer { Serializer::Csv(serializer) => serializer.encode(event, buffer), Serializer::Gelf(serializer) => serializer.encode(event, buffer), Serializer::Json(serializer) => serializer.encode(event, buffer), + Serializer::PrettyJson(serializer) => serializer.encode(event, buffer), Serializer::Logfmt(serializer) => serializer.encode(event, buffer), Serializer::Native(serializer) => serializer.encode(event, buffer), Serializer::NativeJson(serializer) => serializer.encode(event, buffer), diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index 4a124f8987060..c2c2b6e78a49d 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -100,7 +100,7 @@ impl EncodingConfigWithFraming { let framer = match (framer, &serializer) { (Some(framer), _) => framer, - (None, Serializer::Json(_)) => match sink_type { + (None, Serializer::Json(_) | Serializer::PrettyJson(_)) => match sink_type { SinkType::StreamBased => NewlineDelimitedEncoder::new().into(), SinkType::MessageBased => CharacterDelimitedEncoder::new(b',').into(), }, diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index d12f2ab85cb78..17854be598009 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -106,11 +106,11 @@ impl Encoder { /// Get the HTTP content type. pub const fn content_type(&self) -> &'static str { match (&self.serializer, &self.framer) { - (Serializer::Json(_) | Serializer::NativeJson(_), Framer::NewlineDelimited(_)) => { + (Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), Framer::NewlineDelimited(_)) => { "application/x-ndjson" } ( - Serializer::Gelf(_) | Serializer::Json(_) | Serializer::NativeJson(_), + Serializer::Gelf(_) | Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), ) => "application/json", (Serializer::Native(_), _) | (Serializer::Protobuf(_), _) => "application/octet-stream", @@ -121,6 +121,7 @@ impl Encoder { | Serializer::Json(_) | Serializer::Logfmt(_) | Serializer::NativeJson(_) + | Serializer::PrettyJson(_) | Serializer::RawMessage(_) | Serializer::Text(_), _, diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index 82d5ea2b906be..f07e37314721b 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -209,6 +209,7 @@ fn serializer_config_to_deserializer( SerializerConfig::Csv { .. } => todo!(), SerializerConfig::Gelf => DeserializerConfig::Gelf(Default::default()), SerializerConfig::Json(_) => DeserializerConfig::Json(Default::default()), + SerializerConfig::PrettyJson(_) => DeserializerConfig::Json(Default::default()), SerializerConfig::Logfmt => todo!(), SerializerConfig::Native => DeserializerConfig::Native, SerializerConfig::NativeJson => DeserializerConfig::NativeJson(Default::default()), diff --git a/src/sinks/websocket/sink.rs b/src/sinks/websocket/sink.rs index 7c59253c28c6f..91b9318087d4e 100644 --- a/src/sinks/websocket/sink.rs +++ b/src/sinks/websocket/sink.rs @@ -236,12 +236,12 @@ impl WebSocketSink { const fn should_encode_as_binary(&self) -> bool { use vector_lib::codecs::encoding::Serializer::{ - Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, Protobuf, RawMessage, Text, + Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, PrettyJson, Protobuf, RawMessage, Text, }; match self.encoder.serializer() { RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, - Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | Text(_) | NativeJson(_) => false, + Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | PrettyJson(_)| Text(_) | NativeJson(_) => false, } } From c7cd9c8c2ed13cf1aa9904c6515d1f1dd591d2e4 Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Fri, 26 Apr 2024 21:16:40 +0530 Subject: [PATCH 2/8] chore(lint): fix cargo fmt lints --- lib/codecs/src/encoding/format/mod.rs | 4 ++-- lib/codecs/src/encoding/format/pretty_json.rs | 18 +++++++++--------- lib/codecs/src/encoding/mod.rs | 5 ++++- src/codecs/encoding/encoder.rs | 12 ++++++++---- src/sinks/websocket/sink.rs | 7 +++++-- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index d45910bd8651a..76bec3e5728c8 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -11,10 +11,10 @@ mod json; mod logfmt; mod native; mod native_json; +mod pretty_json; mod protobuf; mod raw_message; mod text; -mod pretty_json; use std::fmt::Debug; @@ -23,10 +23,10 @@ pub use avro::{AvroSerializer, AvroSerializerConfig, AvroSerializerOptions}; use dyn_clone::DynClone; pub use gelf::{GelfSerializer, GelfSerializerConfig}; pub use json::{JsonSerializer, JsonSerializerConfig}; -pub use pretty_json::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; pub use logfmt::{LogfmtSerializer, LogfmtSerializerConfig}; pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; +pub use pretty_json::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; pub use text::{TextSerializer, TextSerializerConfig}; diff --git a/lib/codecs/src/encoding/format/pretty_json.rs b/lib/codecs/src/encoding/format/pretty_json.rs index 6f56742fb6fce..20f5058016504 100644 --- a/lib/codecs/src/encoding/format/pretty_json.rs +++ b/lib/codecs/src/encoding/format/pretty_json.rs @@ -4,7 +4,7 @@ use vector_core::{config::DataType, event::Event, schema}; use crate::MetricTagValues; -/// Config used to build a `JsonSerializer`. +/// Config used to build a `PrettyJsonSerializer`. #[crate::configurable_component] #[derive(Debug, Clone, Default)] pub struct PrettyJsonSerializerConfig { @@ -17,17 +17,17 @@ pub struct PrettyJsonSerializerConfig { } impl PrettyJsonSerializerConfig { - /// Creates a new `JsonSerializerConfig`. + /// Creates a new `PrettyJsonSerializerConfig`. pub const fn new(metric_tag_values: MetricTagValues) -> Self { Self { metric_tag_values } } - /// Build the `JsonSerializer` from this configuration. + /// Build the `PrettyJsonSerializer` from this configuration. pub const fn build(&self) -> PrettyJsonSerializer { PrettyJsonSerializer::new(self.metric_tag_values) } - /// The data type of events that are accepted by `JsonSerializer`. + /// The data type of events that are accepted by `PrettyJsonSerializer`. pub fn input_type(&self) -> DataType { DataType::all() } @@ -47,7 +47,7 @@ pub struct PrettyJsonSerializer { } impl PrettyJsonSerializer { - /// Creates a new `JsonSerializer`. + /// Creates a new `PrettyJsonSerializer`. pub const fn new(metric_tag_values: MetricTagValues) -> Self { Self { metric_tag_values } } @@ -99,7 +99,7 @@ mod tests { "z" => Value::from(25), "a" => Value::from("0"), })); - let bytes = serialize(JsonSerializerConfig::default(), event); + let bytes = serialize(PrettyJsonSerializerConfig::default(), event); assert_eq!(bytes, r#"{"a":"0","x":"23","z":25}"#); } @@ -126,7 +126,7 @@ mod tests { )), ); - let bytes = serialize(JsonSerializerConfig::default(), event); + let bytes = serialize(PrettyJsonSerializerConfig::default(), event); assert_eq!( bytes, @@ -144,7 +144,7 @@ mod tests { }, )); - let bytes = serialize(JsonSerializerConfig::default(), event); + let bytes = serialize(PrettyJsonSerializerConfig::default(), event); assert_eq!( bytes, @@ -163,7 +163,7 @@ mod tests { }, )); - let bytes = serialize(JsonSerializerConfig::default(), event); + let bytes = serialize(PrettyJsonSerializerConfig::default(), event); assert_eq!( bytes, diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 36398e54fe314..95afbd7689edf 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -450,7 +450,10 @@ impl Serializer { /// Check if the serializer supports encoding an event to JSON via `Serializer::to_json_value`. pub fn supports_json(&self) -> bool { match self { - Serializer::Json(_) | Self::PrettyJson(_) | Serializer::NativeJson(_) | Serializer::Gelf(_) => true, + Serializer::Json(_) + | Self::PrettyJson(_) + | Serializer::NativeJson(_) + | Serializer::Gelf(_) => true, Serializer::Avro(_) | Serializer::Csv(_) | Serializer::Logfmt(_) diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index 17854be598009..72866baeeea79 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -106,11 +106,15 @@ impl Encoder { /// Get the HTTP content type. pub const fn content_type(&self) -> &'static str { match (&self.serializer, &self.framer) { - (Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), Framer::NewlineDelimited(_)) => { - "application/x-ndjson" - } ( - Serializer::Gelf(_) | Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), + Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), + Framer::NewlineDelimited(_), + ) => "application/x-ndjson", + ( + Serializer::Gelf(_) + | Serializer::Json(_) + | Serializer::PrettyJson(_) + | Serializer::NativeJson(_), Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), ) => "application/json", (Serializer::Native(_), _) | (Serializer::Protobuf(_), _) => "application/octet-stream", diff --git a/src/sinks/websocket/sink.rs b/src/sinks/websocket/sink.rs index 91b9318087d4e..6cb51ad8f0a49 100644 --- a/src/sinks/websocket/sink.rs +++ b/src/sinks/websocket/sink.rs @@ -236,12 +236,15 @@ impl WebSocketSink { const fn should_encode_as_binary(&self) -> bool { use vector_lib::codecs::encoding::Serializer::{ - Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, PrettyJson, Protobuf, RawMessage, Text, + Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, PrettyJson, Protobuf, RawMessage, + Text, }; match self.encoder.serializer() { RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, - Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | PrettyJson(_)| Text(_) | NativeJson(_) => false, + Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | PrettyJson(_) | Text(_) | NativeJson(_) => { + false + } } } From 4c4aa1f31032a31f5f53093f4c4fd5ad65183556 Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Fri, 26 Apr 2024 21:17:37 +0530 Subject: [PATCH 3/8] feat(new codec): update default console logger to use pretty json --- config/vector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/vector.yaml b/config/vector.yaml index fbc56f9de47f8..4c5f2194cd6db 100644 --- a/config/vector.yaml +++ b/config/vector.yaml @@ -37,7 +37,7 @@ sinks: type: "console" inputs: ["parse_logs"] encoding: - codec: "json" + codec: "pretty_json" # Vector's GraphQL API (disabled by default) # Uncomment to try it out with the `vector top` command or From ed49709e3ca80a9873c43098e10c432a9ef845ef Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Fri, 26 Apr 2024 21:18:14 +0530 Subject: [PATCH 4/8] docs(new codec): update docs for pretty json --- website/cue/reference/components/sinks/base/amqp.cue | 7 ++++++- .../components/sinks/base/aws_cloudwatch_logs.cue | 7 ++++++- .../components/sinks/base/aws_kinesis_firehose.cue | 7 ++++++- .../components/sinks/base/aws_kinesis_streams.cue | 7 ++++++- website/cue/reference/components/sinks/base/aws_s3.cue | 7 ++++++- website/cue/reference/components/sinks/base/aws_sns.cue | 7 ++++++- website/cue/reference/components/sinks/base/aws_sqs.cue | 7 ++++++- website/cue/reference/components/sinks/base/azure_blob.cue | 7 ++++++- website/cue/reference/components/sinks/base/console.cue | 7 ++++++- website/cue/reference/components/sinks/base/file.cue | 7 ++++++- .../components/sinks/base/gcp_chronicle_unstructured.cue | 7 ++++++- .../reference/components/sinks/base/gcp_cloud_storage.cue | 7 ++++++- website/cue/reference/components/sinks/base/gcp_pubsub.cue | 7 ++++++- website/cue/reference/components/sinks/base/http.cue | 7 ++++++- website/cue/reference/components/sinks/base/humio_logs.cue | 7 ++++++- website/cue/reference/components/sinks/base/kafka.cue | 7 ++++++- website/cue/reference/components/sinks/base/loki.cue | 7 ++++++- website/cue/reference/components/sinks/base/mqtt.cue | 7 ++++++- website/cue/reference/components/sinks/base/nats.cue | 7 ++++++- website/cue/reference/components/sinks/base/papertrail.cue | 7 ++++++- website/cue/reference/components/sinks/base/pulsar.cue | 7 ++++++- website/cue/reference/components/sinks/base/redis.cue | 7 ++++++- website/cue/reference/components/sinks/base/socket.cue | 7 ++++++- .../reference/components/sinks/base/splunk_hec_logs.cue | 7 ++++++- website/cue/reference/components/sinks/base/webhdfs.cue | 7 ++++++- website/cue/reference/components/sinks/base/websocket.cue | 7 ++++++- 26 files changed, 156 insertions(+), 26 deletions(-) diff --git a/website/cue/reference/components/sinks/base/amqp.cue b/website/cue/reference/components/sinks/base/amqp.cue index 4e8756abec522..8e0954908bc1e 100644 --- a/website/cue/reference/components/sinks/base/amqp.cue +++ b/website/cue/reference/components/sinks/base/amqp.cue @@ -115,6 +115,11 @@ base: components: sinks: amqp: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -235,7 +240,7 @@ base: components: sinks: amqp: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 089e8b08cdbc8..7c93d3796d6a9 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -291,6 +291,11 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -411,7 +416,7 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 10fbde2a50fba..8c76e220c1fb1 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -270,6 +270,11 @@ base: components: sinks: aws_kinesis_firehose: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -390,7 +395,7 @@ base: components: sinks: aws_kinesis_firehose: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index b725956698c9d..c96a948a17a13 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -270,6 +270,11 @@ base: components: sinks: aws_kinesis_streams: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -390,7 +395,7 @@ base: components: sinks: aws_kinesis_streams: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 22cd5fe05c844..bf7467e20f1a0 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -379,6 +379,11 @@ base: components: sinks: aws_s3: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -499,7 +504,7 @@ base: components: sinks: aws_s3: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 80fdfd174727d..eb74eba555e88 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -201,6 +201,11 @@ base: components: sinks: aws_sns: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -321,7 +326,7 @@ base: components: sinks: aws_sns: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index aa19d564cfc27..4024667bab2b5 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -201,6 +201,11 @@ base: components: sinks: aws_sqs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -321,7 +326,7 @@ base: components: sinks: aws_sqs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 6500fd4fe7a04..54f6c213ae861 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -233,6 +233,11 @@ base: components: sinks: azure_blob: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -353,7 +358,7 @@ base: components: sinks: azure_blob: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index 465669604aa88..8843b76f535cc 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -99,6 +99,11 @@ base: components: sinks: console: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -219,7 +224,7 @@ base: components: sinks: console: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index a84ac357af819..9f9aa7d492a95 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -119,6 +119,11 @@ base: components: sinks: file: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -239,7 +244,7 @@ base: components: sinks: file: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index bedd487decbaa..496ffe8ae5120 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -168,6 +168,11 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -288,7 +293,7 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 7a8ff63ec1b6f..030346e5d6872 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -257,6 +257,11 @@ base: components: sinks: gcp_cloud_storage: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -377,7 +382,7 @@ base: components: sinks: gcp_cloud_storage: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index 1361b65080694..fcec8641a6915 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -166,6 +166,11 @@ base: components: sinks: gcp_pubsub: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -286,7 +291,7 @@ base: components: sinks: gcp_pubsub: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index bc886fd29ec61..42ab0caa308ba 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -212,6 +212,11 @@ base: components: sinks: http: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -332,7 +337,7 @@ base: components: sinks: http: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 6c2658f532e23..c4903d4027c8b 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -165,6 +165,11 @@ base: components: sinks: humio_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -285,7 +290,7 @@ base: components: sinks: humio_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index e6870a8515562..5673e0082d416 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -154,6 +154,11 @@ base: components: sinks: kafka: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -274,7 +279,7 @@ base: components: sinks: kafka: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 0594110e84048..6ff1ed7209e8f 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -214,6 +214,11 @@ base: components: sinks: loki: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -334,7 +339,7 @@ base: components: sinks: loki: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/mqtt.cue b/website/cue/reference/components/sinks/base/mqtt.cue index a93de9aa10452..9a77c85327078 100644 --- a/website/cue/reference/components/sinks/base/mqtt.cue +++ b/website/cue/reference/components/sinks/base/mqtt.cue @@ -109,6 +109,11 @@ base: components: sinks: mqtt: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -229,7 +234,7 @@ base: components: sinks: mqtt: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index afd87e7d0f247..50b06591e83d3 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -199,6 +199,11 @@ base: components: sinks: nats: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -319,7 +324,7 @@ base: components: sinks: nats: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/papertrail.cue b/website/cue/reference/components/sinks/base/papertrail.cue index bff6fef8a2516..a2cc5ef18ec5c 100644 --- a/website/cue/reference/components/sinks/base/papertrail.cue +++ b/website/cue/reference/components/sinks/base/papertrail.cue @@ -99,6 +99,11 @@ base: components: sinks: papertrail: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -219,7 +224,7 @@ base: components: sinks: papertrail: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index dd7e31eba9e28..e170ff325ed84 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -193,6 +193,11 @@ base: components: sinks: pulsar: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -313,7 +318,7 @@ base: components: sinks: pulsar: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index 27f2311522011..1b1361db1d2ee 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -152,6 +152,11 @@ base: components: sinks: redis: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -272,7 +277,7 @@ base: components: sinks: redis: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index ed9622fa25366..23154ebe7b388 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -111,6 +111,11 @@ base: components: sinks: socket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -231,7 +236,7 @@ base: components: sinks: socket: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index 660fdd59b0029..ec7141524008f 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -215,6 +215,11 @@ base: components: sinks: splunk_hec_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -335,7 +340,7 @@ base: components: sinks: splunk_hec_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index e570de139f4b4..2c2b0e8bb7db4 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -165,6 +165,11 @@ base: components: sinks: webhdfs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -285,7 +290,7 @@ base: components: sinks: webhdfs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index aa4df23e6e8d3..7aa0f77adaaa2 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -146,6 +146,11 @@ base: components: sinks: websocket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + pretty_json: """ + Encodes an event as [JSON][json] in pretty format. + + [json]: https://www.json.org/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -266,7 +271,7 @@ base: components: sinks: websocket: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" required: false type: string: { default: "single" From 4f875cda3c59ef293a87e6323bdde35d5b235067 Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Sat, 27 Apr 2024 16:20:41 +0530 Subject: [PATCH 5/8] chore(changelog): add changelog.d entry --- changelog.d/20384_pretty_json_config.enhancement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/20384_pretty_json_config.enhancement.md diff --git a/changelog.d/20384_pretty_json_config.enhancement.md b/changelog.d/20384_pretty_json_config.enhancement.md new file mode 100644 index 0000000000000..f4a01e5316c8f --- /dev/null +++ b/changelog.d/20384_pretty_json_config.enhancement.md @@ -0,0 +1,3 @@ +Add `pretty_json` encoding codec to output a prettified json format. + +authors: lsampras From e8bf8fca9098fda7192cdf3ee3e9e24e8e765a8b Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Tue, 30 Apr 2024 10:58:04 +0530 Subject: [PATCH 6/8] fix(tests): update tests for pretty_json codec --- lib/codecs/src/encoding/format/pretty_json.rs | 74 +++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/lib/codecs/src/encoding/format/pretty_json.rs b/lib/codecs/src/encoding/format/pretty_json.rs index 20f5058016504..5d111245c16f8 100644 --- a/lib/codecs/src/encoding/format/pretty_json.rs +++ b/lib/codecs/src/encoding/format/pretty_json.rs @@ -101,7 +101,14 @@ mod tests { })); let bytes = serialize(PrettyJsonSerializerConfig::default(), event); - assert_eq!(bytes, r#"{"a":"0","x":"23","z":25}"#); + assert_eq!( + bytes, + r#"{ + "a": "0", + "x": "23", + "z": 25 +}"# + ); } #[test] @@ -130,7 +137,20 @@ mod tests { assert_eq!( bytes, - r#"{"name":"foos","namespace":"vector","tags":{"Key3":"Value3","key1":"value1","key2":"value2"},"timestamp":"2018-11-14T08:09:10.000000011Z","kind":"incremental","counter":{"value":100.0}}"# + r#"{ + "name": "foos", + "namespace": "vector", + "tags": { + "Key3": "Value3", + "key1": "value1", + "key2": "value2" + }, + "timestamp": "2018-11-14T08:09:10.000000011Z", + "kind": "incremental", + "counter": { + "value": 100.0 + } +}"# ); } @@ -148,7 +168,15 @@ mod tests { assert_eq!( bytes, - r#"{"name":"users","kind":"incremental","set":{"values":["bob"]}}"# + r#"{ + "name": "users", + "kind": "incremental", + "set": { + "values": [ + "bob" + ] + } +}"# ); } @@ -167,7 +195,19 @@ mod tests { assert_eq!( bytes, - r#"{"name":"glork","kind":"incremental","distribution":{"samples":[{"value":10.0,"rate":1}],"statistic":"histogram"}}"# + r#"{ + "name": "glork", + "kind": "incremental", + "distribution": { + "samples": [ + { + "value": 10.0, + "rate": 1 + } + ], + "statistic": "histogram" + } +}"# ); } @@ -197,7 +237,20 @@ mod tests { assert_eq!( bytes, - r#"{"name":"counter","tags":{"a":["first",null,"second"]},"kind":"incremental","counter":{"value":1.0}}"# + r#"{ + "name": "counter", + "tags": { + "a": [ + "first", + null, + "second" + ] + }, + "kind": "incremental", + "counter": { + "value": 1.0 + } +}"# ); } @@ -212,7 +265,16 @@ mod tests { assert_eq!( bytes, - r#"{"name":"counter","tags":{"a":"second"},"kind":"incremental","counter":{"value":1.0}}"# + r#"{ + "name": "counter", + "tags": { + "a": "second" + }, + "kind": "incremental", + "counter": { + "value": 1.0 + } +}"# ); } From 28edc0480e6a5981659ad67c077b46a370732a53 Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Mon, 6 May 2024 21:06:11 +0530 Subject: [PATCH 7/8] refactor(pretty_json): adds a field to existing json codec instead of a new pretty_json codec --- .../20384_pretty_json_config.enhancement.md | 2 +- config/vector.yaml | 4 +- lib/codecs/src/encoding/format/json.rs | 265 ++++++++++++++- lib/codecs/src/encoding/format/mod.rs | 4 +- lib/codecs/src/encoding/format/pretty_json.rs | 301 ------------------ lib/codecs/src/encoding/mod.rs | 20 +- lib/k8s-e2e-tests/tests/vector-agent.rs | 4 + src/codecs/encoding/config.rs | 2 +- src/codecs/encoding/encoder.rs | 13 +- src/components/validation/resources/event.rs | 7 +- src/components/validation/resources/mod.rs | 1 - src/generate.rs | 19 +- src/sinks/datadog/logs/config.rs | 5 +- src/sinks/http/config.rs | 8 +- src/sinks/splunk_hec/logs/config.rs | 8 +- src/sinks/websocket/sink.rs | 7 +- .../reference/components/sinks/base/amqp.cue | 17 +- .../sinks/base/aws_cloudwatch_logs.cue | 17 +- .../sinks/base/aws_kinesis_firehose.cue | 17 +- .../sinks/base/aws_kinesis_streams.cue | 17 +- .../components/sinks/base/aws_s3.cue | 17 +- .../components/sinks/base/aws_sns.cue | 17 +- .../components/sinks/base/aws_sqs.cue | 17 +- .../components/sinks/base/azure_blob.cue | 17 +- .../components/sinks/base/console.cue | 17 +- .../components/sinks/base/databend.cue | 10 + .../reference/components/sinks/base/file.cue | 17 +- .../sinks/base/gcp_chronicle_unstructured.cue | 17 +- .../sinks/base/gcp_cloud_storage.cue | 17 +- .../components/sinks/base/gcp_pubsub.cue | 17 +- .../reference/components/sinks/base/http.cue | 17 +- .../components/sinks/base/humio_logs.cue | 17 +- .../reference/components/sinks/base/kafka.cue | 17 +- .../reference/components/sinks/base/loki.cue | 17 +- .../reference/components/sinks/base/mqtt.cue | 17 +- .../reference/components/sinks/base/nats.cue | 17 +- .../components/sinks/base/papertrail.cue | 17 +- .../components/sinks/base/pulsar.cue | 17 +- .../reference/components/sinks/base/redis.cue | 17 +- .../components/sinks/base/socket.cue | 17 +- .../components/sinks/base/splunk_hec_logs.cue | 17 +- .../components/sinks/base/webhdfs.cue | 17 +- .../components/sinks/base/websocket.cue | 17 +- 43 files changed, 605 insertions(+), 517 deletions(-) delete mode 100644 lib/codecs/src/encoding/format/pretty_json.rs diff --git a/changelog.d/20384_pretty_json_config.enhancement.md b/changelog.d/20384_pretty_json_config.enhancement.md index f4a01e5316c8f..d106fd5f27677 100644 --- a/changelog.d/20384_pretty_json_config.enhancement.md +++ b/changelog.d/20384_pretty_json_config.enhancement.md @@ -1,3 +1,3 @@ -Add `pretty_json` encoding codec to output a prettified json format. +Add `use_pretty_json` option to json coded to output a prettified json format. authors: lsampras diff --git a/config/vector.yaml b/config/vector.yaml index 4c5f2194cd6db..3b8bde0d754d8 100644 --- a/config/vector.yaml +++ b/config/vector.yaml @@ -37,7 +37,9 @@ sinks: type: "console" inputs: ["parse_logs"] encoding: - codec: "pretty_json" + codec: "json" + json: + use_pretty_json: true # Vector's GraphQL API (disabled by default) # Uncomment to try it out with the `vector top` command or diff --git a/lib/codecs/src/encoding/format/json.rs b/lib/codecs/src/encoding/format/json.rs index efd4640b0fe77..81ed0ec5d5fd2 100644 --- a/lib/codecs/src/encoding/format/json.rs +++ b/lib/codecs/src/encoding/format/json.rs @@ -14,17 +14,33 @@ pub struct JsonSerializerConfig { /// metric. When set to `full`, all metric tags are exposed as separate assignments. #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] pub metric_tag_values: MetricTagValues, + + /// Options for the JsonSerializer. + #[serde(default, rename = "json")] + pub options: JsonSerializerOptions, +} + +/// Options for the JsonSerializer. +#[crate::configurable_component] +#[derive(Debug, Clone, Default)] +pub struct JsonSerializerOptions { + /// Whether to use pretty JSON formatting. + #[serde(default)] + pub use_pretty_json: bool, } impl JsonSerializerConfig { /// Creates a new `JsonSerializerConfig`. - pub const fn new(metric_tag_values: MetricTagValues) -> Self { - Self { metric_tag_values } + pub const fn new(metric_tag_values: MetricTagValues, options: JsonSerializerOptions) -> Self { + Self { + metric_tag_values, + options, + } } /// Build the `JsonSerializer` from this configuration. - pub const fn build(&self) -> JsonSerializer { - JsonSerializer::new(self.metric_tag_values) + pub fn build(&self) -> JsonSerializer { + JsonSerializer::new(self.metric_tag_values, self.options.clone()) } /// The data type of events that are accepted by `JsonSerializer`. @@ -44,12 +60,16 @@ impl JsonSerializerConfig { #[derive(Debug, Clone)] pub struct JsonSerializer { metric_tag_values: MetricTagValues, + options: JsonSerializerOptions, } impl JsonSerializer { /// Creates a new `JsonSerializer`. - pub const fn new(metric_tag_values: MetricTagValues) -> Self { - Self { metric_tag_values } + pub const fn new(metric_tag_values: MetricTagValues, options: JsonSerializerOptions) -> Self { + Self { + metric_tag_values, + options, + } } /// Encode event and represent it as JSON value. @@ -68,15 +88,28 @@ impl Encoder for JsonSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let writer = buffer.writer(); - match event { - Event::Log(log) => serde_json::to_writer(writer, &log), - Event::Metric(mut metric) => { - if self.metric_tag_values == MetricTagValues::Single { - metric.reduce_tags_to_single(); + if self.options.use_pretty_json { + match event { + Event::Log(log) => serde_json::to_writer_pretty(writer, &log), + Event::Metric(mut metric) => { + if self.metric_tag_values == MetricTagValues::Single { + metric.reduce_tags_to_single(); + } + serde_json::to_writer_pretty(writer, &metric) } - serde_json::to_writer(writer, &metric) + Event::Trace(trace) => serde_json::to_writer_pretty(writer, &trace), + } + } else { + match event { + Event::Log(log) => serde_json::to_writer(writer, &log), + Event::Metric(mut metric) => { + if self.metric_tag_values == MetricTagValues::Single { + metric.reduce_tags_to_single(); + } + serde_json::to_writer(writer, &metric) + } + Event::Trace(trace) => serde_json::to_writer(writer, &trace), } - Event::Trace(trace) => serde_json::to_writer(writer, &trace), } .map_err(Into::into) } @@ -191,6 +224,7 @@ mod tests { let bytes = serialize( JsonSerializerConfig { metric_tag_values: MetricTagValues::Full, + options: JsonSerializerOptions::default(), }, metric2(), ); @@ -206,6 +240,7 @@ mod tests { let bytes = serialize( JsonSerializerConfig { metric_tag_values: MetricTagValues::Single, + options: JsonSerializerOptions::default(), }, metric2(), ); @@ -236,4 +271,208 @@ mod tests { config.build().encode(input, &mut buffer).unwrap(); buffer.freeze() } + + mod pretty_json { + + use super::*; + use bytes::{Bytes, BytesMut}; + use chrono::{TimeZone, Timelike, Utc}; + use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, StatisticKind, Value}; + use vector_core::metric_tags; + use vrl::btreemap; + + fn get_pretty_json_config() -> JsonSerializerConfig { + JsonSerializerConfig { + options: JsonSerializerOptions { + use_pretty_json: true, + }, + ..Default::default() + } + } + + #[test] + fn serialize_json_log() { + let event = Event::Log(LogEvent::from( + btreemap! {"x" => Value::from("23"),"z" => Value::from(25),"a" => Value::from("0"),}, + )); + let bytes = serialize(get_pretty_json_config(), event); + assert_eq!( + bytes, + r#"{ + "a": "0", + "x": "23", + "z": 25 +}"# + ); + } + #[test] + fn serialize_json_metric_counter() { + let event = Event::Metric( + Metric::new( + "foos", + MetricKind::Incremental, + MetricValue::Counter { value: 100.0 }, + ) + .with_namespace(Some("vector")) + .with_tags(Some( + metric_tags!("key2" => "value2","key1" => "value1","Key3" => "Value3",), + )) + .with_timestamp(Some( + Utc.with_ymd_and_hms(2018, 11, 14, 8, 9, 10) + .single() + .and_then(|t| t.with_nanosecond(11)) + .expect("invalid timestamp"), + )), + ); + let bytes = serialize(get_pretty_json_config(), event); + assert_eq!( + bytes, + r#"{ + "name": "foos", + "namespace": "vector", + "tags": { + "Key3": "Value3", + "key1": "value1", + "key2": "value2" + }, + "timestamp": "2018-11-14T08:09:10.000000011Z", + "kind": "incremental", + "counter": { + "value": 100.0 + } +}"# + ); + } + #[test] + fn serialize_json_metric_set() { + let event = Event::Metric(Metric::new( + "users", + MetricKind::Incremental, + MetricValue::Set { + values: vec!["bob".into()].into_iter().collect(), + }, + )); + let bytes = serialize(get_pretty_json_config(), event); + assert_eq!( + bytes, + r#"{ + "name": "users", + "kind": "incremental", + "set": { + "values": [ + "bob" + ] + } +}"# + ); + } + #[test] + fn serialize_json_metric_histogram_without_timestamp() { + let event = Event::Metric(Metric::new( + "glork", + MetricKind::Incremental, + MetricValue::Distribution { + samples: vector_core::samples![10.0 => 1], + statistic: StatisticKind::Histogram, + }, + )); + let bytes = serialize(get_pretty_json_config(), event); + assert_eq!( + bytes, + r#"{ + "name": "glork", + "kind": "incremental", + "distribution": { + "samples": [ + { + "value": 10.0, + "rate": 1 + } + ], + "statistic": "histogram" + } +}"# + ); + } + #[test] + fn serialize_equals_to_json_value() { + let event = Event::Log(LogEvent::from(btreemap! {"foo" => Value::from("bar")})); + let mut serializer = get_pretty_json_config().build(); + let mut bytes = BytesMut::new(); + serializer.encode(event.clone(), &mut bytes).unwrap(); + let json = serializer.to_json_value(event).unwrap(); + assert_eq!(bytes.freeze(), serde_json::to_string_pretty(&json).unwrap()); + } + #[test] + fn serialize_metric_tags_full() { + let bytes = serialize( + JsonSerializerConfig { + metric_tag_values: MetricTagValues::Full, + options: JsonSerializerOptions { + use_pretty_json: true, + }, + }, + metric2(), + ); + assert_eq!( + bytes, + r#"{ + "name": "counter", + "tags": { + "a": [ + "first", + null, + "second" + ] + }, + "kind": "incremental", + "counter": { + "value": 1.0 + } +}"# + ); + } + #[test] + fn serialize_metric_tags_single() { + let bytes = serialize( + JsonSerializerConfig { + metric_tag_values: MetricTagValues::Single, + options: JsonSerializerOptions { + use_pretty_json: true, + }, + }, + metric2(), + ); + assert_eq!( + bytes, + r#"{ + "name": "counter", + "tags": { + "a": "second" + }, + "kind": "incremental", + "counter": { + "value": 1.0 + } +}"# + ); + } + fn metric2() -> Event { + Event::Metric( + Metric::new( + "counter", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + ) + .with_tags(Some( + metric_tags! ("a" => "first","a" => None,"a" => "second",), + )), + ) + } + fn serialize(config: JsonSerializerConfig, input: Event) -> Bytes { + let mut buffer = BytesMut::new(); + config.build().encode(input, &mut buffer).unwrap(); + buffer.freeze() + } + } } diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 76bec3e5728c8..01317e243cca6 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -11,7 +11,6 @@ mod json; mod logfmt; mod native; mod native_json; -mod pretty_json; mod protobuf; mod raw_message; mod text; @@ -22,11 +21,10 @@ pub use self::csv::{CsvSerializer, CsvSerializerConfig}; pub use avro::{AvroSerializer, AvroSerializerConfig, AvroSerializerOptions}; use dyn_clone::DynClone; pub use gelf::{GelfSerializer, GelfSerializerConfig}; -pub use json::{JsonSerializer, JsonSerializerConfig}; +pub use json::{JsonSerializer, JsonSerializerConfig, JsonSerializerOptions}; pub use logfmt::{LogfmtSerializer, LogfmtSerializerConfig}; pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; -pub use pretty_json::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; pub use text::{TextSerializer, TextSerializerConfig}; diff --git a/lib/codecs/src/encoding/format/pretty_json.rs b/lib/codecs/src/encoding/format/pretty_json.rs deleted file mode 100644 index 5d111245c16f8..0000000000000 --- a/lib/codecs/src/encoding/format/pretty_json.rs +++ /dev/null @@ -1,301 +0,0 @@ -use bytes::{BufMut, BytesMut}; -use tokio_util::codec::Encoder; -use vector_core::{config::DataType, event::Event, schema}; - -use crate::MetricTagValues; - -/// Config used to build a `PrettyJsonSerializer`. -#[crate::configurable_component] -#[derive(Debug, Clone, Default)] -pub struct PrettyJsonSerializerConfig { - /// Controls how metric tag values are encoded. - /// - /// When set to `single`, only the last non-bare value of tags are displayed with the - /// metric. When set to `full`, all metric tags are exposed as separate assignments. - #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] - pub metric_tag_values: MetricTagValues, -} - -impl PrettyJsonSerializerConfig { - /// Creates a new `PrettyJsonSerializerConfig`. - pub const fn new(metric_tag_values: MetricTagValues) -> Self { - Self { metric_tag_values } - } - - /// Build the `PrettyJsonSerializer` from this configuration. - pub const fn build(&self) -> PrettyJsonSerializer { - PrettyJsonSerializer::new(self.metric_tag_values) - } - - /// The data type of events that are accepted by `PrettyJsonSerializer`. - pub fn input_type(&self) -> DataType { - DataType::all() - } - - /// The schema required by the serializer. - pub fn schema_requirement(&self) -> schema::Requirement { - // While technically we support `Value` variants that can't be losslessly serialized to - // PrettyJson, we don't want to enforce that limitation to users yet. - schema::Requirement::empty() - } -} - -/// Serializer that converts an `Event` to bytes using the PrettyJson format. -#[derive(Debug, Clone)] -pub struct PrettyJsonSerializer { - metric_tag_values: MetricTagValues, -} - -impl PrettyJsonSerializer { - /// Creates a new `PrettyJsonSerializer`. - pub const fn new(metric_tag_values: MetricTagValues) -> Self { - Self { metric_tag_values } - } - - /// Encode event and represent it as PrettyJson value. - pub fn to_json_value(&self, event: Event) -> Result { - match event { - Event::Log(log) => serde_json::to_value(&log), - Event::Metric(metric) => serde_json::to_value(&metric), - Event::Trace(trace) => serde_json::to_value(&trace), - } - .map_err(|e| e.to_string().into()) - } -} - -impl Encoder for PrettyJsonSerializer { - type Error = vector_common::Error; - - fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { - let writer = buffer.writer(); - match event { - Event::Log(log) => serde_json::to_writer_pretty(writer, &log), - Event::Metric(mut metric) => { - if self.metric_tag_values == MetricTagValues::Single { - metric.reduce_tags_to_single(); - } - serde_json::to_writer_pretty(writer, &metric) - } - Event::Trace(trace) => serde_json::to_writer_pretty(writer, &trace), - } - .map_err(Into::into) - } -} - -#[cfg(test)] -mod tests { - use bytes::{Bytes, BytesMut}; - use chrono::{TimeZone, Timelike, Utc}; - use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, StatisticKind, Value}; - use vector_core::metric_tags; - use vrl::btreemap; - - use super::*; - - #[test] - fn serialize_json_log() { - let event = Event::Log(LogEvent::from(btreemap! { - "x" => Value::from("23"), - "z" => Value::from(25), - "a" => Value::from("0"), - })); - let bytes = serialize(PrettyJsonSerializerConfig::default(), event); - - assert_eq!( - bytes, - r#"{ - "a": "0", - "x": "23", - "z": 25 -}"# - ); - } - - #[test] - fn serialize_json_metric_counter() { - let event = Event::Metric( - Metric::new( - "foos", - MetricKind::Incremental, - MetricValue::Counter { value: 100.0 }, - ) - .with_namespace(Some("vector")) - .with_tags(Some(metric_tags!( - "key2" => "value2", - "key1" => "value1", - "Key3" => "Value3", - ))) - .with_timestamp(Some( - Utc.with_ymd_and_hms(2018, 11, 14, 8, 9, 10) - .single() - .and_then(|t| t.with_nanosecond(11)) - .expect("invalid timestamp"), - )), - ); - - let bytes = serialize(PrettyJsonSerializerConfig::default(), event); - - assert_eq!( - bytes, - r#"{ - "name": "foos", - "namespace": "vector", - "tags": { - "Key3": "Value3", - "key1": "value1", - "key2": "value2" - }, - "timestamp": "2018-11-14T08:09:10.000000011Z", - "kind": "incremental", - "counter": { - "value": 100.0 - } -}"# - ); - } - - #[test] - fn serialize_json_metric_set() { - let event = Event::Metric(Metric::new( - "users", - MetricKind::Incremental, - MetricValue::Set { - values: vec!["bob".into()].into_iter().collect(), - }, - )); - - let bytes = serialize(PrettyJsonSerializerConfig::default(), event); - - assert_eq!( - bytes, - r#"{ - "name": "users", - "kind": "incremental", - "set": { - "values": [ - "bob" - ] - } -}"# - ); - } - - #[test] - fn serialize_json_metric_histogram_without_timestamp() { - let event = Event::Metric(Metric::new( - "glork", - MetricKind::Incremental, - MetricValue::Distribution { - samples: vector_core::samples![10.0 => 1], - statistic: StatisticKind::Histogram, - }, - )); - - let bytes = serialize(PrettyJsonSerializerConfig::default(), event); - - assert_eq!( - bytes, - r#"{ - "name": "glork", - "kind": "incremental", - "distribution": { - "samples": [ - { - "value": 10.0, - "rate": 1 - } - ], - "statistic": "histogram" - } -}"# - ); - } - - #[test] - fn serialize_equals_to_json_value() { - let event = Event::Log(LogEvent::from(btreemap! { - "foo" => Value::from("bar") - })); - let mut serializer = PrettyJsonSerializerConfig::default().build(); - let mut bytes = BytesMut::new(); - - serializer.encode(event.clone(), &mut bytes).unwrap(); - - let json = serializer.to_json_value(event).unwrap(); - - assert_eq!(bytes.freeze(), serde_json::to_string_pretty(&json).unwrap()); - } - - #[test] - fn serialize_metric_tags_full() { - let bytes = serialize( - PrettyJsonSerializerConfig { - metric_tag_values: MetricTagValues::Full, - }, - metric2(), - ); - - assert_eq!( - bytes, - r#"{ - "name": "counter", - "tags": { - "a": [ - "first", - null, - "second" - ] - }, - "kind": "incremental", - "counter": { - "value": 1.0 - } -}"# - ); - } - - #[test] - fn serialize_metric_tags_single() { - let bytes = serialize( - PrettyJsonSerializerConfig { - metric_tag_values: MetricTagValues::Single, - }, - metric2(), - ); - - assert_eq!( - bytes, - r#"{ - "name": "counter", - "tags": { - "a": "second" - }, - "kind": "incremental", - "counter": { - "value": 1.0 - } -}"# - ); - } - - fn metric2() -> Event { - Event::Metric( - Metric::new( - "counter", - MetricKind::Incremental, - MetricValue::Counter { value: 1.0 }, - ) - .with_tags(Some(metric_tags! ( - "a" => "first", - "a" => None, - "a" => "second", - ))), - ) - } - - fn serialize(config: PrettyJsonSerializerConfig, input: Event) -> Bytes { - let mut buffer = BytesMut::new(); - config.build().encode(input, &mut buffer).unwrap(); - buffer.freeze() - } -} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 95afbd7689edf..0d766f73e4d17 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -23,8 +23,6 @@ pub use framing::{ use vector_config::configurable_component; use vector_core::{config::DataType, event::Event, schema}; -use self::format::{PrettyJsonSerializer, PrettyJsonSerializerConfig}; - /// An error that occurred while building an encoder. pub type BuildError = Box; @@ -215,11 +213,6 @@ pub enum SerializerConfig { /// [json]: https://www.json.org/ Json(JsonSerializerConfig), - /// Encodes an event as [JSON][json] in pretty format. - /// - /// [json]: https://www.json.org/ - PrettyJson(PrettyJsonSerializerConfig), - /// Encodes an event as a [logfmt][logfmt] message. /// /// [logfmt]: https://brandur.org/logfmt @@ -336,7 +329,6 @@ impl SerializerConfig { SerializerConfig::Csv(config) => Ok(Serializer::Csv(config.build()?)), SerializerConfig::Gelf => Ok(Serializer::Gelf(GelfSerializerConfig::new().build())), SerializerConfig::Json(config) => Ok(Serializer::Json(config.build())), - SerializerConfig::PrettyJson(config) => Ok(Serializer::PrettyJson(config.build())), SerializerConfig::Logfmt => Ok(Serializer::Logfmt(LogfmtSerializerConfig.build())), SerializerConfig::Native => Ok(Serializer::Native(NativeSerializerConfig.build())), SerializerConfig::NativeJson => { @@ -372,7 +364,6 @@ impl SerializerConfig { SerializerConfig::Csv(_) | SerializerConfig::Gelf | SerializerConfig::Json(_) - | SerializerConfig::PrettyJson(_) | SerializerConfig::Logfmt | SerializerConfig::NativeJson | SerializerConfig::RawMessage @@ -389,7 +380,6 @@ impl SerializerConfig { SerializerConfig::Csv(config) => config.input_type(), SerializerConfig::Gelf { .. } => GelfSerializerConfig::input_type(), SerializerConfig::Json(config) => config.input_type(), - SerializerConfig::PrettyJson(config) => config.input_type(), SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.input_type(), @@ -408,7 +398,6 @@ impl SerializerConfig { SerializerConfig::Csv(config) => config.schema_requirement(), SerializerConfig::Gelf { .. } => GelfSerializerConfig::schema_requirement(), SerializerConfig::Json(config) => config.schema_requirement(), - SerializerConfig::PrettyJson(config) => config.schema_requirement(), SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.schema_requirement(), @@ -430,8 +419,6 @@ pub enum Serializer { Gelf(GelfSerializer), /// Uses a `JsonSerializer` for serialization. Json(JsonSerializer), - /// Uses a `JsonSerializer` for serialization. - PrettyJson(PrettyJsonSerializer), /// Uses a `LogfmtSerializer` for serialization. Logfmt(LogfmtSerializer), /// Uses a `NativeSerializer` for serialization. @@ -450,10 +437,7 @@ impl Serializer { /// Check if the serializer supports encoding an event to JSON via `Serializer::to_json_value`. pub fn supports_json(&self) -> bool { match self { - Serializer::Json(_) - | Self::PrettyJson(_) - | Serializer::NativeJson(_) - | Serializer::Gelf(_) => true, + Serializer::Json(_) | Serializer::NativeJson(_) | Serializer::Gelf(_) => true, Serializer::Avro(_) | Serializer::Csv(_) | Serializer::Logfmt(_) @@ -474,7 +458,6 @@ impl Serializer { match self { Serializer::Gelf(serializer) => serializer.to_json_value(event), Serializer::Json(serializer) => serializer.to_json_value(event), - Serializer::PrettyJson(serializer) => serializer.to_json_value(event), Serializer::NativeJson(serializer) => serializer.to_json_value(event), Serializer::Avro(_) | Serializer::Csv(_) @@ -558,7 +541,6 @@ impl tokio_util::codec::Encoder for Serializer { Serializer::Csv(serializer) => serializer.encode(event, buffer), Serializer::Gelf(serializer) => serializer.encode(event, buffer), Serializer::Json(serializer) => serializer.encode(event, buffer), - Serializer::PrettyJson(serializer) => serializer.encode(event, buffer), Serializer::Logfmt(serializer) => serializer.encode(event, buffer), Serializer::Native(serializer) => serializer.encode(event, buffer), Serializer::NativeJson(serializer) => serializer.encode(event, buffer), diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index e15b690ed74e6..706c8ef5f2a0c 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -993,6 +993,8 @@ async fn custom_selectors() -> Result<(), Box> { inputs: [kubernetes_logs] encoding: codec: json + json: + use_pretty_json: false "#}; let vector = framework @@ -1395,6 +1397,8 @@ async fn glob_pattern_filtering() -> Result<(), Box> { inputs: [kubernetes_logs] encoding: codec: json + json: + use_pretty_json: false "#}; let vector = framework diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index c2c2b6e78a49d..4a124f8987060 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -100,7 +100,7 @@ impl EncodingConfigWithFraming { let framer = match (framer, &serializer) { (Some(framer), _) => framer, - (None, Serializer::Json(_) | Serializer::PrettyJson(_)) => match sink_type { + (None, Serializer::Json(_)) => match sink_type { SinkType::StreamBased => NewlineDelimitedEncoder::new().into(), SinkType::MessageBased => CharacterDelimitedEncoder::new(b',').into(), }, diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index 72866baeeea79..d12f2ab85cb78 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -106,15 +106,11 @@ impl Encoder { /// Get the HTTP content type. pub const fn content_type(&self) -> &'static str { match (&self.serializer, &self.framer) { + (Serializer::Json(_) | Serializer::NativeJson(_), Framer::NewlineDelimited(_)) => { + "application/x-ndjson" + } ( - Serializer::Json(_) | Serializer::PrettyJson(_) | Serializer::NativeJson(_), - Framer::NewlineDelimited(_), - ) => "application/x-ndjson", - ( - Serializer::Gelf(_) - | Serializer::Json(_) - | Serializer::PrettyJson(_) - | Serializer::NativeJson(_), + Serializer::Gelf(_) | Serializer::Json(_) | Serializer::NativeJson(_), Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), ) => "application/json", (Serializer::Native(_), _) | (Serializer::Protobuf(_), _) => "application/octet-stream", @@ -125,7 +121,6 @@ impl Encoder { | Serializer::Json(_) | Serializer::Logfmt(_) | Serializer::NativeJson(_) - | Serializer::PrettyJson(_) | Serializer::RawMessage(_) | Serializer::Text(_), _, diff --git a/src/components/validation/resources/event.rs b/src/components/validation/resources/event.rs index fa4564a6076c0..a0ec8d594bae4 100644 --- a/src/components/validation/resources/event.rs +++ b/src/components/validation/resources/event.rs @@ -5,6 +5,7 @@ use serde::Deserialize; use serde_json::Value; use snafu::Snafu; use tokio_util::codec::Encoder as _; +use vector_lib::codecs::encoding::format::JsonSerializerOptions; use crate::codecs::Encoder; use vector_lib::codecs::{ @@ -179,7 +180,11 @@ pub fn encode_test_event( } else { Encoder::::new( NewlineDelimitedEncoder::new().into(), - JsonSerializer::new(MetricTagValues::default()).into(), + JsonSerializer::new( + MetricTagValues::default(), + JsonSerializerOptions::default(), + ) + .into(), ) }; diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index f07e37314721b..82d5ea2b906be 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -209,7 +209,6 @@ fn serializer_config_to_deserializer( SerializerConfig::Csv { .. } => todo!(), SerializerConfig::Gelf => DeserializerConfig::Gelf(Default::default()), SerializerConfig::Json(_) => DeserializerConfig::Json(Default::default()), - SerializerConfig::PrettyJson(_) => DeserializerConfig::Json(Default::default()), SerializerConfig::Logfmt => todo!(), SerializerConfig::Native => DeserializerConfig::Native, SerializerConfig::NativeJson => DeserializerConfig::NativeJson(Default::default()), diff --git a/src/generate.rs b/src/generate.rs index 09e3ba88d15f5..2070ea6f35cc6 100644 --- a/src/generate.rs +++ b/src/generate.rs @@ -489,6 +489,9 @@ mod tests { [sinks.sink0.encoding] codec = "json" + [sinks.sink0.encoding.json] + use_pretty_json = false + [sinks.sink0.healthcheck] enabled = true @@ -526,6 +529,9 @@ mod tests { [sinks.sink0.encoding] codec = "json" + [sinks.sink0.encoding.json] + use_pretty_json = false + [sinks.sink0.healthcheck] enabled = true @@ -557,6 +563,9 @@ mod tests { [sinks.sink0.encoding] codec = "json" + [sinks.sink0.encoding.json] + use_pretty_json = false + [sinks.sink0.healthcheck] enabled = true @@ -581,6 +590,9 @@ mod tests { [sinks.sink0.encoding] codec = "json" + [sinks.sink0.encoding.json] + use_pretty_json = false + [sinks.sink0.healthcheck] enabled = true @@ -691,6 +703,8 @@ mod tests { type: console encoding: codec: json + json: + use_pretty_json: false healthcheck: enabled: true uri: null @@ -756,7 +770,10 @@ mod tests { "target": "stdout", "type": "console", "encoding": { - "codec": "json" + "codec": "json", + "json": { + "use_pretty_json": false + } }, "healthcheck": { "enabled": true, diff --git a/src/sinks/datadog/logs/config.rs b/src/sinks/datadog/logs/config.rs index 3f83f5733fe55..09ab362929330 100644 --- a/src/sinks/datadog/logs/config.rs +++ b/src/sinks/datadog/logs/config.rs @@ -186,7 +186,7 @@ mod test { use crate::codecs::EncodingConfigWithFraming; use crate::components::validation::prelude::*; use vector_lib::{ - codecs::{JsonSerializerConfig, MetricTagValues}, + codecs::{encoding::format::JsonSerializerOptions, JsonSerializerConfig, MetricTagValues}, config::LogNamespace, }; @@ -209,7 +209,8 @@ mod test { let encoding = EncodingConfigWithFraming::new( None, - JsonSerializerConfig::new(MetricTagValues::Full).into(), + JsonSerializerConfig::new(MetricTagValues::Full, JsonSerializerOptions::default()) + .into(), config.encoding.clone(), ); diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index e025ab81b5a84..ccc9ace780046 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -308,6 +308,8 @@ impl SinkConfig for HttpSinkConfig { #[cfg(test)] mod tests { + use vector_lib::codecs::encoding::format::JsonSerializerOptions; + use super::*; use crate::components::validation::prelude::*; @@ -323,7 +325,11 @@ mod tests { method: HttpMethod::Post, encoding: EncodingConfigWithFraming::new( None, - JsonSerializerConfig::new(MetricTagValues::Full).into(), + JsonSerializerConfig::new( + MetricTagValues::Full, + JsonSerializerOptions::default(), + ) + .into(), Transformer::default(), ), auth: None, diff --git a/src/sinks/splunk_hec/logs/config.rs b/src/sinks/splunk_hec/logs/config.rs index 844d073245bad..d229e45691106 100644 --- a/src/sinks/splunk_hec/logs/config.rs +++ b/src/sinks/splunk_hec/logs/config.rs @@ -291,7 +291,7 @@ mod tests { use super::*; use crate::components::validation::prelude::*; use vector_lib::{ - codecs::{JsonSerializerConfig, MetricTagValues}, + codecs::{encoding::format::JsonSerializerOptions, JsonSerializerConfig, MetricTagValues}, config::LogNamespace, }; @@ -316,7 +316,11 @@ mod tests { sourcetype: None, source: None, encoding: EncodingConfig::new( - JsonSerializerConfig::new(MetricTagValues::Full).into(), + JsonSerializerConfig::new( + MetricTagValues::Full, + JsonSerializerOptions::default(), + ) + .into(), Transformer::default(), ), compression: Compression::default(), diff --git a/src/sinks/websocket/sink.rs b/src/sinks/websocket/sink.rs index 6cb51ad8f0a49..7c59253c28c6f 100644 --- a/src/sinks/websocket/sink.rs +++ b/src/sinks/websocket/sink.rs @@ -236,15 +236,12 @@ impl WebSocketSink { const fn should_encode_as_binary(&self) -> bool { use vector_lib::codecs::encoding::Serializer::{ - Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, PrettyJson, Protobuf, RawMessage, - Text, + Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, Protobuf, RawMessage, Text, }; match self.encoder.serializer() { RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, - Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | PrettyJson(_) | Text(_) | NativeJson(_) => { - false - } + Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | Text(_) | NativeJson(_) => false, } } diff --git a/website/cue/reference/components/sinks/base/amqp.cue b/website/cue/reference/components/sinks/base/amqp.cue index 8e0954908bc1e..0fbc20fe1ab2c 100644 --- a/website/cue/reference/components/sinks/base/amqp.cue +++ b/website/cue/reference/components/sinks/base/amqp.cue @@ -115,11 +115,6 @@ base: components: sinks: amqp: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -233,6 +228,16 @@ base: components: sinks: amqp: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -240,7 +245,7 @@ base: components: sinks: amqp: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 7c93d3796d6a9..8b4ac2e87d047 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -291,11 +291,6 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -409,6 +404,16 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -416,7 +421,7 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 8c76e220c1fb1..9a5fa7a7db8e5 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -270,11 +270,6 @@ base: components: sinks: aws_kinesis_firehose: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -388,6 +383,16 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -395,7 +400,7 @@ base: components: sinks: aws_kinesis_firehose: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index c96a948a17a13..e98fd05f2c907 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -270,11 +270,6 @@ base: components: sinks: aws_kinesis_streams: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -388,6 +383,16 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -395,7 +400,7 @@ base: components: sinks: aws_kinesis_streams: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index bf7467e20f1a0..55f83e7d257a9 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -379,11 +379,6 @@ base: components: sinks: aws_s3: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -497,6 +492,16 @@ base: components: sinks: aws_s3: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -504,7 +509,7 @@ base: components: sinks: aws_s3: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index eb74eba555e88..7d020b7965356 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -201,11 +201,6 @@ base: components: sinks: aws_sns: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -319,6 +314,16 @@ base: components: sinks: aws_sns: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -326,7 +331,7 @@ base: components: sinks: aws_sns: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 4024667bab2b5..0d780de6158e6 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -201,11 +201,6 @@ base: components: sinks: aws_sqs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -319,6 +314,16 @@ base: components: sinks: aws_sqs: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -326,7 +331,7 @@ base: components: sinks: aws_sqs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 54f6c213ae861..f2979d70057df 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -233,11 +233,6 @@ base: components: sinks: azure_blob: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -351,6 +346,16 @@ base: components: sinks: azure_blob: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -358,7 +363,7 @@ base: components: sinks: azure_blob: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index 8843b76f535cc..43f4c1c474cd7 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -99,11 +99,6 @@ base: components: sinks: console: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -217,6 +212,16 @@ base: components: sinks: console: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -224,7 +229,7 @@ base: components: sinks: console: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 3b9b9ffe1a8b6..003077d2827fa 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -231,6 +231,16 @@ base: components: sinks: databend: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index 9f9aa7d492a95..06df564aac74b 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -119,11 +119,6 @@ base: components: sinks: file: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -237,6 +232,16 @@ base: components: sinks: file: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -244,7 +249,7 @@ base: components: sinks: file: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index 496ffe8ae5120..becc4fe502d34 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -168,11 +168,6 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -286,6 +281,16 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -293,7 +298,7 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 030346e5d6872..0442754f47b45 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -257,11 +257,6 @@ base: components: sinks: gcp_cloud_storage: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -375,6 +370,16 @@ base: components: sinks: gcp_cloud_storage: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -382,7 +387,7 @@ base: components: sinks: gcp_cloud_storage: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index fcec8641a6915..69a08dd60add1 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -166,11 +166,6 @@ base: components: sinks: gcp_pubsub: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -284,6 +279,16 @@ base: components: sinks: gcp_pubsub: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -291,7 +296,7 @@ base: components: sinks: gcp_pubsub: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 42ab0caa308ba..4798a5dfd3a5e 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -212,11 +212,6 @@ base: components: sinks: http: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -330,6 +325,16 @@ base: components: sinks: http: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -337,7 +342,7 @@ base: components: sinks: http: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index c4903d4027c8b..1e9680dabff46 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -165,11 +165,6 @@ base: components: sinks: humio_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -283,6 +278,16 @@ base: components: sinks: humio_logs: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -290,7 +295,7 @@ base: components: sinks: humio_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index 5673e0082d416..69fa756ef4d7f 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -154,11 +154,6 @@ base: components: sinks: kafka: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -272,6 +267,16 @@ base: components: sinks: kafka: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -279,7 +284,7 @@ base: components: sinks: kafka: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 6ff1ed7209e8f..10c97fb673d6b 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -214,11 +214,6 @@ base: components: sinks: loki: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -332,6 +327,16 @@ base: components: sinks: loki: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -339,7 +344,7 @@ base: components: sinks: loki: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/mqtt.cue b/website/cue/reference/components/sinks/base/mqtt.cue index 9a77c85327078..c7387ccbf2da6 100644 --- a/website/cue/reference/components/sinks/base/mqtt.cue +++ b/website/cue/reference/components/sinks/base/mqtt.cue @@ -109,11 +109,6 @@ base: components: sinks: mqtt: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -227,6 +222,16 @@ base: components: sinks: mqtt: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -234,7 +239,7 @@ base: components: sinks: mqtt: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index 50b06591e83d3..ea19afd6333f1 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -199,11 +199,6 @@ base: components: sinks: nats: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -317,6 +312,16 @@ base: components: sinks: nats: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -324,7 +329,7 @@ base: components: sinks: nats: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/papertrail.cue b/website/cue/reference/components/sinks/base/papertrail.cue index a2cc5ef18ec5c..f7c53831e4fca 100644 --- a/website/cue/reference/components/sinks/base/papertrail.cue +++ b/website/cue/reference/components/sinks/base/papertrail.cue @@ -99,11 +99,6 @@ base: components: sinks: papertrail: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -217,6 +212,16 @@ base: components: sinks: papertrail: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -224,7 +229,7 @@ base: components: sinks: papertrail: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index e170ff325ed84..087072d4e3194 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -193,11 +193,6 @@ base: components: sinks: pulsar: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -311,6 +306,16 @@ base: components: sinks: pulsar: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -318,7 +323,7 @@ base: components: sinks: pulsar: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index 1b1361db1d2ee..adbb2f4812131 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -152,11 +152,6 @@ base: components: sinks: redis: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -270,6 +265,16 @@ base: components: sinks: redis: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -277,7 +282,7 @@ base: components: sinks: redis: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index 23154ebe7b388..e65b8d2a10654 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -111,11 +111,6 @@ base: components: sinks: socket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -229,6 +224,16 @@ base: components: sinks: socket: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -236,7 +241,7 @@ base: components: sinks: socket: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index ec7141524008f..1be843dcd52ec 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -215,11 +215,6 @@ base: components: sinks: splunk_hec_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -333,6 +328,16 @@ base: components: sinks: splunk_hec_logs: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -340,7 +345,7 @@ base: components: sinks: splunk_hec_logs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index 2c2b0e8bb7db4..041bc2e14f29a 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -165,11 +165,6 @@ base: components: sinks: webhdfs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -283,6 +278,16 @@ base: components: sinks: webhdfs: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -290,7 +295,7 @@ base: components: sinks: webhdfs: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index 7aa0f77adaaa2..7208cf0edd09e 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -146,11 +146,6 @@ base: components: sinks: websocket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ - pretty_json: """ - Encodes an event as [JSON][json] in pretty format. - - [json]: https://www.json.org/ - """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. @@ -264,6 +259,16 @@ base: components: sinks: websocket: configuration: { required: false type: array: items: type: string: {} } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: use_pretty_json: { + description: "Whether to use pretty JSON formatting." + required: false + type: bool: default: false + } + } metric_tag_values: { description: """ Controls how metric tag values are encoded. @@ -271,7 +276,7 @@ base: components: sinks: websocket: configuration: { When set to `single`, only the last non-bare value of tags are displayed with the metric. When set to `full`, all metric tags are exposed as separate assignments. """ - relevant_when: "codec = \"json\" or codec = \"pretty_json\" or codec = \"text\"" + relevant_when: "codec = \"json\" or codec = \"text\"" required: false type: string: { default: "single" From 5535612416ee8f259546e14fcfb59e24f9ea89ec Mon Sep 17 00:00:00 2001 From: Sampras Lopes Date: Wed, 8 May 2024 01:04:29 +0530 Subject: [PATCH 8/8] refactor(codecs): rename pretty field name for json codec --- .../20384_pretty_json_config.enhancement.md | 2 +- config/vector.yaml | 2 +- lib/codecs/src/encoding/format/json.rs | 16 +++++----------- lib/k8s-e2e-tests/tests/vector-agent.rs | 4 ++-- src/generate.rs | 12 ++++++------ .../cue/reference/components/sinks/base/amqp.cue | 2 +- .../sinks/base/aws_cloudwatch_logs.cue | 2 +- .../sinks/base/aws_kinesis_firehose.cue | 2 +- .../sinks/base/aws_kinesis_streams.cue | 2 +- .../reference/components/sinks/base/aws_s3.cue | 2 +- .../reference/components/sinks/base/aws_sns.cue | 2 +- .../reference/components/sinks/base/aws_sqs.cue | 2 +- .../components/sinks/base/azure_blob.cue | 2 +- .../reference/components/sinks/base/console.cue | 2 +- .../reference/components/sinks/base/databend.cue | 2 +- .../cue/reference/components/sinks/base/file.cue | 2 +- .../sinks/base/gcp_chronicle_unstructured.cue | 2 +- .../components/sinks/base/gcp_cloud_storage.cue | 2 +- .../components/sinks/base/gcp_pubsub.cue | 2 +- .../cue/reference/components/sinks/base/http.cue | 2 +- .../components/sinks/base/humio_logs.cue | 2 +- .../reference/components/sinks/base/kafka.cue | 2 +- .../cue/reference/components/sinks/base/loki.cue | 2 +- .../cue/reference/components/sinks/base/mqtt.cue | 2 +- .../cue/reference/components/sinks/base/nats.cue | 2 +- .../components/sinks/base/papertrail.cue | 2 +- .../reference/components/sinks/base/pulsar.cue | 2 +- .../reference/components/sinks/base/redis.cue | 2 +- .../reference/components/sinks/base/socket.cue | 2 +- .../components/sinks/base/splunk_hec_logs.cue | 2 +- .../reference/components/sinks/base/webhdfs.cue | 2 +- .../components/sinks/base/websocket.cue | 2 +- 32 files changed, 42 insertions(+), 48 deletions(-) diff --git a/changelog.d/20384_pretty_json_config.enhancement.md b/changelog.d/20384_pretty_json_config.enhancement.md index d106fd5f27677..002d2ce7e00b5 100644 --- a/changelog.d/20384_pretty_json_config.enhancement.md +++ b/changelog.d/20384_pretty_json_config.enhancement.md @@ -1,3 +1,3 @@ -Add `use_pretty_json` option to json coded to output a prettified json format. +Add `pretty` option to json coded to output a prettified json format. authors: lsampras diff --git a/config/vector.yaml b/config/vector.yaml index 3b8bde0d754d8..590c2aa9ba152 100644 --- a/config/vector.yaml +++ b/config/vector.yaml @@ -39,7 +39,7 @@ sinks: encoding: codec: "json" json: - use_pretty_json: true + pretty: true # Vector's GraphQL API (disabled by default) # Uncomment to try it out with the `vector top` command or diff --git a/lib/codecs/src/encoding/format/json.rs b/lib/codecs/src/encoding/format/json.rs index 81ed0ec5d5fd2..a72d3473c8d55 100644 --- a/lib/codecs/src/encoding/format/json.rs +++ b/lib/codecs/src/encoding/format/json.rs @@ -26,7 +26,7 @@ pub struct JsonSerializerConfig { pub struct JsonSerializerOptions { /// Whether to use pretty JSON formatting. #[serde(default)] - pub use_pretty_json: bool, + pub pretty: bool, } impl JsonSerializerConfig { @@ -88,7 +88,7 @@ impl Encoder for JsonSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let writer = buffer.writer(); - if self.options.use_pretty_json { + if self.options.pretty { match event { Event::Log(log) => serde_json::to_writer_pretty(writer, &log), Event::Metric(mut metric) => { @@ -283,9 +283,7 @@ mod tests { fn get_pretty_json_config() -> JsonSerializerConfig { JsonSerializerConfig { - options: JsonSerializerOptions { - use_pretty_json: true, - }, + options: JsonSerializerOptions { pretty: true }, ..Default::default() } } @@ -408,9 +406,7 @@ mod tests { let bytes = serialize( JsonSerializerConfig { metric_tag_values: MetricTagValues::Full, - options: JsonSerializerOptions { - use_pretty_json: true, - }, + options: JsonSerializerOptions { pretty: true }, }, metric2(), ); @@ -437,9 +433,7 @@ mod tests { let bytes = serialize( JsonSerializerConfig { metric_tag_values: MetricTagValues::Single, - options: JsonSerializerOptions { - use_pretty_json: true, - }, + options: JsonSerializerOptions { pretty: true }, }, metric2(), ); diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 706c8ef5f2a0c..49bd9a2439d29 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -994,7 +994,7 @@ async fn custom_selectors() -> Result<(), Box> { encoding: codec: json json: - use_pretty_json: false + pretty: false "#}; let vector = framework @@ -1398,7 +1398,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { encoding: codec: json json: - use_pretty_json: false + pretty: false "#}; let vector = framework diff --git a/src/generate.rs b/src/generate.rs index 2070ea6f35cc6..d143ee2fb3e13 100644 --- a/src/generate.rs +++ b/src/generate.rs @@ -490,7 +490,7 @@ mod tests { codec = "json" [sinks.sink0.encoding.json] - use_pretty_json = false + pretty = false [sinks.sink0.healthcheck] enabled = true @@ -530,7 +530,7 @@ mod tests { codec = "json" [sinks.sink0.encoding.json] - use_pretty_json = false + pretty = false [sinks.sink0.healthcheck] enabled = true @@ -564,7 +564,7 @@ mod tests { codec = "json" [sinks.sink0.encoding.json] - use_pretty_json = false + pretty = false [sinks.sink0.healthcheck] enabled = true @@ -591,7 +591,7 @@ mod tests { codec = "json" [sinks.sink0.encoding.json] - use_pretty_json = false + pretty = false [sinks.sink0.healthcheck] enabled = true @@ -704,7 +704,7 @@ mod tests { encoding: codec: json json: - use_pretty_json: false + pretty: false healthcheck: enabled: true uri: null @@ -772,7 +772,7 @@ mod tests { "encoding": { "codec": "json", "json": { - "use_pretty_json": false + "pretty": false } }, "healthcheck": { diff --git a/website/cue/reference/components/sinks/base/amqp.cue b/website/cue/reference/components/sinks/base/amqp.cue index 0fbc20fe1ab2c..f6636096e5875 100644 --- a/website/cue/reference/components/sinks/base/amqp.cue +++ b/website/cue/reference/components/sinks/base/amqp.cue @@ -232,7 +232,7 @@ base: components: sinks: amqp: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 8b4ac2e87d047..214248b03ba3b 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -408,7 +408,7 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 9a5fa7a7db8e5..d752b12153a25 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -387,7 +387,7 @@ base: components: sinks: aws_kinesis_firehose: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index e98fd05f2c907..a683d120eb3be 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -387,7 +387,7 @@ base: components: sinks: aws_kinesis_streams: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 55f83e7d257a9..e04ba5e1c0451 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -496,7 +496,7 @@ base: components: sinks: aws_s3: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 7d020b7965356..c4798bbe1e2cb 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -318,7 +318,7 @@ base: components: sinks: aws_sns: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 0d780de6158e6..88440bd495915 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -318,7 +318,7 @@ base: components: sinks: aws_sqs: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index f2979d70057df..bc0afd1f92535 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -350,7 +350,7 @@ base: components: sinks: azure_blob: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index 43f4c1c474cd7..1082c6a2a85eb 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -216,7 +216,7 @@ base: components: sinks: console: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 003077d2827fa..97611c0f75915 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -235,7 +235,7 @@ base: components: sinks: databend: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index 06df564aac74b..eb83f9673e973 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -236,7 +236,7 @@ base: components: sinks: file: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index becc4fe502d34..79bfc5acf4eda 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -285,7 +285,7 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 0442754f47b45..22b86cfe60cae 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -374,7 +374,7 @@ base: components: sinks: gcp_cloud_storage: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index 69a08dd60add1..f132ee545598b 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -283,7 +283,7 @@ base: components: sinks: gcp_pubsub: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 4798a5dfd3a5e..c28aa0a594e43 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -329,7 +329,7 @@ base: components: sinks: http: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 1e9680dabff46..5eb9352aa509e 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -282,7 +282,7 @@ base: components: sinks: humio_logs: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index 69fa756ef4d7f..54cac4cdc33fb 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -271,7 +271,7 @@ base: components: sinks: kafka: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 10c97fb673d6b..fe83109cac514 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -331,7 +331,7 @@ base: components: sinks: loki: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/mqtt.cue b/website/cue/reference/components/sinks/base/mqtt.cue index c7387ccbf2da6..74c7cebc160c2 100644 --- a/website/cue/reference/components/sinks/base/mqtt.cue +++ b/website/cue/reference/components/sinks/base/mqtt.cue @@ -226,7 +226,7 @@ base: components: sinks: mqtt: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index ea19afd6333f1..175089fdf9324 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -316,7 +316,7 @@ base: components: sinks: nats: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/papertrail.cue b/website/cue/reference/components/sinks/base/papertrail.cue index f7c53831e4fca..b3131ca8f635d 100644 --- a/website/cue/reference/components/sinks/base/papertrail.cue +++ b/website/cue/reference/components/sinks/base/papertrail.cue @@ -216,7 +216,7 @@ base: components: sinks: papertrail: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index 087072d4e3194..9dc1adfe7e212 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -310,7 +310,7 @@ base: components: sinks: pulsar: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index adbb2f4812131..7b82723ab71c7 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -269,7 +269,7 @@ base: components: sinks: redis: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index e65b8d2a10654..0e0f0f604a293 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -228,7 +228,7 @@ base: components: sinks: socket: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index 1be843dcd52ec..c0f238141d084 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -332,7 +332,7 @@ base: components: sinks: splunk_hec_logs: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index 041bc2e14f29a..75e2fcd19005f 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -282,7 +282,7 @@ base: components: sinks: webhdfs: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index 7208cf0edd09e..13a9dd8eeadee 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -263,7 +263,7 @@ base: components: sinks: websocket: configuration: { description: "Options for the JsonSerializer." relevant_when: "codec = \"json\"" required: false - type: object: options: use_pretty_json: { + type: object: options: pretty: { description: "Whether to use pretty JSON formatting." required: false type: bool: default: false