enhancement(sinks): Allow Encoding config to only/except list fields

.meta/_partials/_encoding.toml

@@ -0,0 +1,53 @@
+[<%= namespace %>.encoding]
+type = "table"
+common = true
+description = """\
+Configures the encoding specific sink behavior.\
+"""
+
+[<%= namespace %>.encoding.children.format]
+type = "string"
+common = true
+<% if default %>
+required = false
+default = <%= default.to_toml %>
+examples = <%= encodings.to_toml %>
+<% else %>
+required = true
+examples = <%= encodings.to_toml %>
+<% end %>
+description = """\
+The encoding format used to serialize the events before outputting.\
+"""
+
+[<%= namespace %>.encoding.children.format.enum]
+<%- if encodings.include?("text") -%>text = "Each event is encoded into text via the `message` key and the payload is new line delimited."<%- end -%>
+<%- if encodings.include?("json") -%>json = "Each event is encoded into JSON and the payload is represented as a JSON array."<%- end -%>
+<%- if encodings.include?("ndjson") -%>ndjson = "Each event is encoded into JSON and the payload is new line delimited."<%- end -%>
+<%- if encodings.include?("default") -%>default = "Each event is encoded according to the default for this sink."<%- end -%>
+
+[<%= namespace %>.encoding.children.only_fields]
+type = "[string]"
+required = false
+examples = [["timestamp", "message", "host"]]
+description = """\
+Limit the sink to only encoding the specified labels.\
+"""
+
+[<%= namespace %>.encoding.children.except_fields]
+type = "[string]"
+required = false
+examples = [["timestamp", "message", "host"]]
+description = """\
+Prevent the sink from encoding the specified labels.\
+"""
+
+[<%= namespace %>.encoding.children.timestamp_format]
+type = "string"
+required = false
+default = "rfc3339"
+description = "How to format event timestamps."
+
+[<%= namespace %>.encoding.children.timestamp_format.enum]
+rfc3339 = "Format as an RFC3339 string"
+unix = "Format as a unix timestamp, can be parsed as a Clickhouse DateTime"

.meta/sinks/aws_cloudwatch_logs.toml

@@ -35,18 +35,11 @@ write_to_description = "[Amazon Web Service's CloudWatch Logs service][urls.aws_
   timeout_secs: 30
 ) %>
 
-[sinks.aws_cloudwatch_logs.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.aws_cloudwatch_logs.options.encoding.enum]
-json = "Each event is encoded into JSON and the payload is represented as a JSON array."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.aws_cloudwatch_logs.options",
+  encodings: ["json", "text"],
+  default: "text"
+) %>
 
 [sinks.aws_cloudwatch_logs.options.group_name]
 type = "string"

.meta/sinks/aws_kinesis_firehose.toml

@@ -35,18 +35,11 @@ write_to_description = "[Amazon Web Service's Kinesis Data Firehose][urls.aws_ki
   timeout_secs: 30
 ) %>
 
-[sinks.aws_kinesis_firehose.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.aws_kinesis_firehose.options.encoding.enum]
-json = "Each event is encoded into JSON and the payload is represented as a JSON array."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.aws_kinesis_firehose.options",
+  encodings: ["json", "text"],
+  default: "text"
+) %>
 
 [sinks.aws_kinesis_firehose.options.stream_name]
 type = "string"

.meta/sinks/aws_kinesis_streams.toml

@@ -35,18 +35,11 @@ write_to_description = "[Amazon Web Service's Kinesis Data Stream service][urls.
   timeout_secs: 30
 ) %>
 
-[sinks.aws_kinesis_streams.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.aws_kinesis_streams.options.encoding.enum]
-json = "Each event is encoded into JSON and the payload is represented as a JSON array."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.aws_kinesis_streams.options",
+  encodings: ["json", "text"],
+  default: "text"
+) %>
 
 [sinks.aws_kinesis_streams.options.partition_key_field]
 type = "string"

.meta/sinks/aws_s3.toml

@@ -42,18 +42,10 @@ examples = ["my-bucket"]
 required = true
 description = "The S3 bucket name. Do not include a leading `s3://` or a trailing `/`."
 
-[sinks.aws_s3.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.aws_s3.options.encoding.enum]
-ndjson = "Each event is encoded into JSON and the payload is new line delimited."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.aws_s3.options",
+  encodings: ["ndjson", "text"], default: false
+) %>
 
 [sinks.aws_s3.options.filename_append_uuid]
 type = "bool"

.meta/sinks/clickhouse.toml

@@ -94,19 +94,8 @@ common = true
 examples = ["mydatabase"]
 description = "The database that contains the stable that data will be inserted into."
 
-[sinks.clickhouse.options.encoding]
-type = "table"
-common = true
-required = false
-description = "Customize how events are encoded."
-
-[sinks.clickhouse.options.encoding.children.timestamp_format]
-type = "string"
-common = true
-required = false
-default = "rfc3339"
-description = "How to format event timestamps. Formats such as unix can be parsed as a Clickhouse DateTime, however, this loses precision as DateTimes are defined in seconds."
-
-[sinks.clickhouse.options.encoding.children.timestamp_format.enum]
-rfc3339 = "Format as an RFC3339 string"
-unix = "Format as a unix timestamp, can be parsed as a Clickhouse DateTime"
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.clickhouse.options",
+  encodings: ["default"],
+  default: "default"
+) %>

.meta/sinks/console.toml

@@ -12,18 +12,7 @@ write_to_description = "[standard output streams][urls.standard_streams], such a
 
 <%= render("_partials/_component_options.toml", type: "sink", name: "console") %>
 
-[sinks.console.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.console.options.encoding.enum]
-json = "Each event is encoded into JSON."
-text = "Each event is encoded into text via the `message` key."
+<%= render("_partials/_encoding.toml", namespace: "sinks.console.options", encodings: ["json", "text"], default: false) %>
 
 [sinks.console.options.target]
 type = "string"

.meta/sinks/elasticsearch.toml

@@ -33,6 +33,12 @@ write_to_description = "[Elasticsearch][urls.elasticsearch] via the [`_bulk` API
   timeout_secs: 60
 ) %>
 
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.elasticsearch.options",
+  encodings: ["default"],
+  default: "default"
+) %>
+
 [sinks.elasticsearch.options.auth]
 type = "table"
 common = false

.meta/sinks/file.toml

@@ -12,18 +12,11 @@ write_to_description = "a file"
 
 <%= render("_partials/_component_options.toml", type: "sink", name: "file") %>
 
-[sinks.file.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.file.options.encoding.enum]
-ndjson = "Each event is encoded into JSON and the payload is new line delimited."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.file.options",
+  encodings: ["text", "ndjson"],
+  default: false
+) %>
 
 [sinks.file.options.path]
 type = "string"

.meta/sinks/gcp_cloud_storage.toml

@@ -116,16 +116,11 @@ default = "log"
 required = true
 description = "The filename extension to use in the object name."
 
-[sinks.gcp_cloud_storage.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = "The encoding format used to serialize the events before outputting."
-
-[sinks.gcp_cloud_storage.options.encoding.enum]
-ndjson = "Each event is encoded into JSON and the payload is new line delimited."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.gcp_cloud_storage.options",
+  encodings: ["ndjson", "text"],
+  default: "text"
+) %>
 
 [sinks.gcp_cloud_storage.options.compression]
 type = "string"

.meta/sinks/gcp_pubsub.toml

@@ -55,3 +55,5 @@ common = false
 required = true
 examples = ["this-is-a-topic"]
 description = "The topic within the project to which to publish logs."
+
+<%= render("_partials/_encoding.toml", namespace: "sinks.gcp_pubsub.options", encodings: ["default"], default: "default") %>

.meta/sinks/gcp_stackdriver_logging.toml

@@ -122,3 +122,9 @@ For example, Compute Engine VM instances use the labels `projectId`, \
 """
 
 <%= render("_partials/_tls_connector_options.toml", namespace: "sinks.gcp_stackdriver_logging.options", can_enable: false, can_verify: true) %>
+
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.gcp_stackdriver_logging.options",
+  encodings: ["default"],
+  default: "default"
+) %>

.meta/sinks/http.toml

@@ -71,19 +71,11 @@ outputting.\
 none = "The payload will not be compressed."
 gzip = "The payload will be compressed in [Gzip][urls.gzip] format before being sent."
 
-[sinks.http.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.http.options.encoding.enum]
-json = "Each event is encoded into JSON and the payload is represented as a JSON array."
-ndjson = "Each event is encoded into JSON and the payload is new line delimited."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.http.options",
+  encodings: ["json", "ndjson", "text"],
+  default: "text"
+) %>
 
 [sinks.http.options.headers]
 type = "table"

.meta/sinks/humio_logs.toml

@@ -41,3 +41,9 @@ type = "string"
 examples = ["http://myhumiohost.com"]
 required = false
 description = "The optional host to send Humio logs to."
+
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.humio_logs.options",
+  encodings: ["json"],
+  default: "json"
+) %>

.meta/sinks/kafka.toml

@@ -26,19 +26,11 @@ A comma delimited list of host and port pairs that the Kafka client should conta
 its cluster metadata.\
 """
 
-[sinks.kafka.options.encoding]
-type = "string"
-category = "requests"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.kafka.options.encoding.enum]
-json = "Each event is encoded into JSON and the payload is represented as a JSON array."
-text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
-
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.kafka.options",
+  encodings: ["json", "text"],
+  default: false
+) %>
 
 [sinks.kafka.options.key_field]
 type = "string"

.meta/sinks/logdna.toml

@@ -71,3 +71,9 @@ required = false
 examples = ["vector", "myapp"]
 default = "vector"
 description = "The default app that will be set for events that do not contain a `file` or `app` field."
+
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.logdna.options",
+  encodings: ["default"],
+  default: "default"
+) %>

.meta/sinks/loki.toml

@@ -38,17 +38,11 @@ required = true
 examples = ["http://localhost:3100", "http://127.0.0.1:8080"]
 description = "The endpoint used to ship logs to."
 
-[sinks.loki.options.encoding]
-type = "string"
-common = true
-required = true
-description = """\
-The encoding format used to serialize the events before outputting.\
-"""
-
-[sinks.loki.options.encoding.enum]
-json = "Each event is encoded into JSON"
-text = "Each event is encoded into text via the `message` key."
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.loki.options",
+  encodings: ["json", "text"],
+  default: "json"
+) %>
 
 [sinks.loki.options.tenant_id]
 type = "string"

.meta/sinks/new_relic_logs.toml

@@ -51,3 +51,9 @@ description = "The API region to send logs to."
 [sinks.new_relic_logs.options.region.enum]
 us = "The US region"
 eu = "The EU region"
+
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.new_relic_logs.options",
+  encodings: ["json"],
+  default: "json"
+) %>

.meta/sinks/sematext.toml

@@ -46,3 +46,9 @@ type = "string"
 required = true
 examples = ["${SEMATEXT_TOKEN_ENV_VAR}", "some-sematext-token"]
 description = "The token that will be used to write to Sematext."
+
+<%= render("_partials/_encoding.toml",
+  namespace: "sinks.sematext_logs.options",
+  encodings: ["default"],
+  default: "default"
+) %>