enhancement(sinks): add compression option for aws sinks (vectordotde…

…v#2953)

* enhancement(aws_kinesis_firehose sink): add compression option

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* enhancement(aws_kinesis_stream sink): add compression option

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* aws: simplify create_client fn

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* enhancement(aws_cloudwatch_logs sink): add compression option

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* enhancement(aws_cloudwatch_metrics sink): add compression option

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* adjust docs

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* add rusoto_core::ContentEncoding conversion

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

* move create_client/healthcheck methods

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>

.meta/sinks/aws_cloudwatch_logs.toml.erb

@@ -53,6 +53,13 @@ write_to_description = "[Amazon Web Service's CloudWatch Logs service][urls.aws_
   encodings: ["json", "text"]
 ) %>
 
+<%= render("_partials/fields/_compression_options.toml",
+  namespace: "sinks.aws_cloudwatch_logs.options",
+  options: {
+    "default" => "none"
+  }
+) %>
+
 [sinks.aws_cloudwatch_logs.options.group_name]
 type = "string"
 common = true

.meta/sinks/aws_cloudwatch_metrics.toml.erb

@@ -28,6 +28,13 @@ write_to_description = "[Amazon Web Service's CloudWatch Metrics service][urls.a
 
 <%= render("_partials/fields/_component_options.toml", type: "sink", name: "aws_cloudwatch_metrics") %>
 
+<%= render("_partials/fields/_compression_options.toml",
+  namespace: "sinks.aws_cloudwatch_metrics.options",
+  options: {
+    "default" => "none"
+  }
+) %>
+
 [sinks.aws_cloudwatch_metrics.options.namespace]
 type = "string"
 common = true

.meta/sinks/aws_kinesis_firehose.toml.erb

@@ -56,6 +56,13 @@ write_to_description = "[Amazon Web Service's Kinesis Data Firehose][urls.aws_ki
   encodings: ["json", "text"]
 ) %>
 
+<%= render("_partials/fields/_compression_options.toml",
+  namespace: "sinks.aws_kinesis_firehose.options",
+  options: {
+    "default" => "none"
+  }
+) %>
+
 [sinks.aws_kinesis_firehose.options.stream_name]
 type = "string"
 common = true

.meta/sinks/aws_kinesis_streams.toml.erb

@@ -56,6 +56,13 @@ write_to_description = "[Amazon Web Service's Kinesis Data Stream service][urls.
   encodings: ["json", "text"]
 ) %>
 
+<%= render("_partials/fields/_compression_options.toml",
+  namespace: "sinks.aws_kinesis_streams.options",
+  options: {
+    "default" => "none"
+  }
+) %>
+
 [sinks.aws_kinesis_streams.options.partition_key_field]
 type = "string"
 common = true

Cargo.toml

@@ -66,7 +66,7 @@ metrics-core = "0.5.2"
 metrics-runtime = "0.13.0"
 
 # Aws
-rusoto_core = { version = "0.44.0", optional = true }
+rusoto_core = { version = "0.44.0", features = ["encoding"], optional = true }
 rusoto_s3 = { version = "0.44.0", optional = true }
 rusoto_logs = { version = "0.44.0", optional = true }
 rusoto_cloudwatch = { version = "0.44.0", optional = true }

config/vector.spec.toml

@@ -2874,6 +2874,16 @@ require('custom_module')
   # * type: string
   assume_role = "arn:aws:iam::123456789098:role/my_role"
 
+  # The compression strategy used to compress the encoded event data before
+  # transmission.
+  #
+  # * optional
+  # * default: "none"
+  # * type: string
+  # * enum: "none" or "gzip"
+  compression = "none"
+  compression = "gzip"
+
   # Dynamically create a log group if it does not already exist. This will ignore
   # `create_missing_stream` directly after creating the group and will create the
   # first stream.
@@ -3130,6 +3140,16 @@ require('custom_module')
   # * type: string
   assume_role = "arn:aws:iam::123456789098:role/my_role"
 
+  # The compression strategy used to compress the encoded event data before
+  # transmission.
+  #
+  # * optional
+  # * default: "none"
+  # * type: string
+  # * enum: "none" or "gzip"
+  compression = "none"
+  compression = "gzip"
+
   # Custom endpoint for use with AWS-compatible services. Providing a value for
   # this option will make `region` moot.
   #
@@ -3209,6 +3229,16 @@ require('custom_module')
   # * type: string
   assume_role = "arn:aws:iam::123456789098:role/my_role"
 
+  # The compression strategy used to compress the encoded event data before
+  # transmission.
+  #
+  # * optional
+  # * default: "none"
+  # * type: string
+  # * enum: "none" or "gzip"
+  compression = "none"
+  compression = "gzip"
+
   # Custom endpoint for use with AWS-compatible services. Providing a value for
   # this option will make `region` moot.
   #
@@ -3430,6 +3460,16 @@ require('custom_module')
   # * type: string
   assume_role = "arn:aws:iam::123456789098:role/my_role"
 
+  # The compression strategy used to compress the encoded event data before
+  # transmission.
+  #
+  # * optional
+  # * default: "none"
+  # * type: string
+  # * enum: "none" or "gzip"
+  compression = "none"
+  compression = "gzip"
+
   # Custom endpoint for use with AWS-compatible services. Providing a value for
   # this option will make `region` moot.
   #