diff --git a/apis/fluentbit/v1alpha2/clusteroutput_types.go b/apis/fluentbit/v1alpha2/clusteroutput_types.go index d6feeb12a..264abfe82 100644 --- a/apis/fluentbit/v1alpha2/clusteroutput_types.go +++ b/apis/fluentbit/v1alpha2/clusteroutput_types.go @@ -47,6 +47,8 @@ type OutputSpec struct { AzureBlob *output.AzureBlob `json:"azureBlob,omitempty"` // AzureLogAnalytics defines AzureLogAnalytics Output Configuration AzureLogAnalytics *output.AzureLogAnalytics `json:"azureLogAnalytics,omitempty"` + // CloudWatch defines CloudWatch Output Configuration + CloudWatch *output.CloudWatch `json:"cloudWatch,omitempty"` // RetryLimit represents configuration for the scheduler which can be set independently on each output section. // This option allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. RetryLimit string `json:"retry_limit,omitempty"` diff --git a/apis/fluentbit/v1alpha2/plugins/output/cloudwatch_types.go b/apis/fluentbit/v1alpha2/plugins/output/cloudwatch_types.go new file mode 100644 index 000000000..fd5896376 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/cloudwatch_types.go @@ -0,0 +1,111 @@ +package output + +import ( + "fmt" + "github.com/fluent/fluent-operator/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// CloudWatch is the AWS CloudWatch output plugin, allows you to ingest your records into AWS CloudWatch. +type CloudWatch struct { + // AWS Region + Region string `json:"region"` + // Name of Cloudwatch Log Group to send log records to + LogGroupName string `json:"logGroupName,omitempty"` + // Template for Log Group name, overrides LogGroupName if set. + LogGroupTemplate string `json:"logGroupTemplate,omitempty"` + // The name of the CloudWatch Log Stream to send log records to + LogStreamName string `json:"logStreamName,omitempty"` + // Prefix for the Log Stream name. Not compatible with LogStreamName setting + LogStreamPrefix string `json:"logStreamPrefix,omitempty"` + // Template for Log Stream name. Overrides LogStreamPrefix and LogStreamName if set. + LogStreamTemplate string `json:"logStreamTemplate,omitempty"` + // If set, only the value of the key will be sent to CloudWatch + LogKey string `json:"logKey,omitempty"` + // Optional parameter to tell CloudWatch the format of the data + LogFormat string `json:"logFormat,omitempty"` + // Role ARN to use for cross-account access + RoleArn string `json:"roleArn,omitempty"` + // Automatically create the log group. Defaults to False. + AutoCreateGroup *bool `json:"autoCreateGroup,omitempty"` + // Number of days logs are retained for + // +kubebuilder:validation:Enum:=1;3;5;7;14;30;60;90;120;150;180;365;400;545;731;1827;3653 + LogRetentionDays *int32 `json:"logRetentionDays,omitempty"` + // Custom endpoint for CloudWatch logs API + Endpoint string `json:"endpoint,omitempty"` + // Optional string to represent the CloudWatch namespace. + MetricNamespace string `json:"metricNamespace,omitempty"` + // Optional lists of lists for dimension keys to be added to all metrics. Use comma separated strings + // for one list of dimensions and semicolon separated strings for list of lists dimensions. + MetricDimensions string `json:"metricDimensions,omitempty"` + // Specify a custom STS endpoint for the AWS STS API + StsEndpoint string `json:"stsEndpoint,omitempty"` + // Automatically retry failed requests to CloudWatch once. Defaults to True. + AutoRetryRequests *bool `json:"autoRetryRequests,omitempty"` + // Specify an external ID for the STS API. + ExternalID string `json:"externalID,omitempty"` +} + +// Name implement Section() method +func (_ *CloudWatch) Name() string { + return "cloudwatch_logs" +} + +// Params implement Section() method +func (o *CloudWatch) Params(sl plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + if o.Region != "" { + kvs.Insert("region", o.Region) + } + if o.LogGroupName != "" { + kvs.Insert("log_group_name", o.LogGroupName) + } + if o.LogGroupTemplate != "" { + kvs.Insert("log_group_template", o.LogGroupTemplate) + } + if o.LogStreamName != "" { + kvs.Insert("log_stream_name", o.LogStreamName) + } + if o.LogStreamPrefix != "" { + kvs.Insert("log_stream_prefix", o.LogStreamPrefix) + } + if o.LogStreamTemplate != "" { + kvs.Insert("log_stream_template", o.LogStreamTemplate) + } + if o.LogKey != "" { + kvs.Insert("log_key", o.LogKey) + } + if o.LogFormat != "" { + kvs.Insert("log_format", o.LogFormat) + } + if o.AutoCreateGroup != nil { + kvs.Insert("auto_create_group", fmt.Sprint(*o.AutoCreateGroup)) + } + if o.LogRetentionDays != nil { + kvs.Insert("log_retention_days", fmt.Sprint(*o.LogRetentionDays)) + } + if o.RoleArn != "" { + kvs.Insert("role_arn", o.RoleArn) + } + if o.Endpoint != "" { + kvs.Insert("endpoint", o.Endpoint) + } + if o.MetricNamespace != "" { + kvs.Insert("metric_namespace", o.MetricNamespace) + } + if o.MetricDimensions != "" { + kvs.Insert("metric_dimensions", o.MetricDimensions) + } + if o.StsEndpoint != "" { + kvs.Insert("sts_endpoint", o.StsEndpoint) + } + if o.AutoRetryRequests != nil { + kvs.Insert("auto_retry_requests", fmt.Sprint(*o.AutoRetryRequests)) + } + if o.ExternalID != "" { + kvs.Insert("external_id", o.ExternalID) + } + return kvs, nil +} diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml index 813bdc5fd..d19fb7dd6 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -177,6 +177,93 @@ spec: - customerID - sharedKey type: object + cloudWatch: + description: CloudWatch defines CloudWatch Output Configuration + properties: + autoCreateGroup: + description: Automatically create the log group. Defaults to False. + type: boolean + autoRetryRequests: + description: Automatically retry failed requests to CloudWatch + once. Defaults to True. + type: boolean + endpoint: + description: Custom endpoint for CloudWatch logs API + type: string + externalID: + description: Specify an external ID for the STS API. + type: string + logFormat: + description: Optional parameter to tell CloudWatch the format + of the data + type: string + logGroupName: + description: Name of Cloudwatch Log Group to send log records + to + type: string + logGroupTemplate: + description: Template for Log Group name, overrides LogGroupName + if set. + type: string + logKey: + description: If set, only the value of the key will be sent to + CloudWatch + type: string + logRetentionDays: + description: Number of days logs are retained for + enum: + - 1 + - 3 + - 5 + - 7 + - 14 + - 30 + - 60 + - 90 + - 120 + - 150 + - 180 + - 365 + - 400 + - 545 + - 731 + - 1827 + - 3653 + format: int32 + type: integer + logStreamName: + description: The name of the CloudWatch Log Stream to send log + records to + type: string + logStreamPrefix: + description: Prefix for the Log Stream name. Not compatible with + LogStreamName setting + type: string + logStreamTemplate: + description: Template for Log Stream name. Overrides LogStreamPrefix + and LogStreamName if set. + type: string + metricDimensions: + description: Optional lists of lists for dimension keys to be + added to all metrics. Use comma separated strings for one list + of dimensions and semicolon separated strings for list of lists + dimensions. + type: string + metricNamespace: + description: Optional string to represent the CloudWatch namespace. + type: string + region: + description: AWS Region + type: string + roleArn: + description: Role ARN to use for cross-account access + type: string + stsEndpoint: + description: Specify a custom STS endpoint for the AWS STS API + type: string + required: + - region + type: object customPlugin: description: CustomPlugin defines Custom Output configuration. properties: diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 813bdc5fd..d19fb7dd6 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -177,6 +177,93 @@ spec: - customerID - sharedKey type: object + cloudWatch: + description: CloudWatch defines CloudWatch Output Configuration + properties: + autoCreateGroup: + description: Automatically create the log group. Defaults to False. + type: boolean + autoRetryRequests: + description: Automatically retry failed requests to CloudWatch + once. Defaults to True. + type: boolean + endpoint: + description: Custom endpoint for CloudWatch logs API + type: string + externalID: + description: Specify an external ID for the STS API. + type: string + logFormat: + description: Optional parameter to tell CloudWatch the format + of the data + type: string + logGroupName: + description: Name of Cloudwatch Log Group to send log records + to + type: string + logGroupTemplate: + description: Template for Log Group name, overrides LogGroupName + if set. + type: string + logKey: + description: If set, only the value of the key will be sent to + CloudWatch + type: string + logRetentionDays: + description: Number of days logs are retained for + enum: + - 1 + - 3 + - 5 + - 7 + - 14 + - 30 + - 60 + - 90 + - 120 + - 150 + - 180 + - 365 + - 400 + - 545 + - 731 + - 1827 + - 3653 + format: int32 + type: integer + logStreamName: + description: The name of the CloudWatch Log Stream to send log + records to + type: string + logStreamPrefix: + description: Prefix for the Log Stream name. Not compatible with + LogStreamName setting + type: string + logStreamTemplate: + description: Template for Log Stream name. Overrides LogStreamPrefix + and LogStreamName if set. + type: string + metricDimensions: + description: Optional lists of lists for dimension keys to be + added to all metrics. Use comma separated strings for one list + of dimensions and semicolon separated strings for list of lists + dimensions. + type: string + metricNamespace: + description: Optional string to represent the CloudWatch namespace. + type: string + region: + description: AWS Region + type: string + roleArn: + description: Role ARN to use for cross-account access + type: string + stsEndpoint: + description: Specify a custom STS endpoint for the AWS STS API + type: string + required: + - region + type: object customPlugin: description: CustomPlugin defines Custom Output configuration. properties: diff --git a/docs/fluentbit.md b/docs/fluentbit.md index f533652d4..03a02afa9 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -341,6 +341,7 @@ OutputSpec defines the desired state of ClusterOutput | alias | A user friendly alias name for this output plugin. Used in metrics for distinction of each configured output. | string | | azureBlob | AzureBlob defines AzureBlob Output Configuration | *[output.AzureBlob](plugins/output/azureblob.md) | | azureLogAnalytics | AzureLogAnalytics defines AzureLogAnalytics Output Configuration | *[output.AzureLogAnalytics](plugins/output/azureloganalytics.md) | +| cloudWatch | CloudWatch defines CloudWatch Output Configuration | *[output.CloudWatch](plugins/output/cloudwatch.md) | | retry_limit | RetryLimit represents configuration for the scheduler which can be set independently on each output section. This option allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. | string | | es | Elasticsearch defines Elasticsearch Output configuration. | *[output.Elasticsearch](plugins/output/elasticsearch.md) | | file | File defines File Output configuration. | *[output.File](plugins/output/file.md) | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index e5fbc20d6..e5a65d52a 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -2179,6 +2179,93 @@ spec: - customerID - sharedKey type: object + cloudWatch: + description: CloudWatch defines CloudWatch Output Configuration + properties: + autoCreateGroup: + description: Automatically create the log group. Defaults to False. + type: boolean + autoRetryRequests: + description: Automatically retry failed requests to CloudWatch + once. Defaults to True. + type: boolean + endpoint: + description: Custom endpoint for CloudWatch logs API + type: string + externalID: + description: Specify an external ID for the STS API. + type: string + logFormat: + description: Optional parameter to tell CloudWatch the format + of the data + type: string + logGroupName: + description: Name of Cloudwatch Log Group to send log records + to + type: string + logGroupTemplate: + description: Template for Log Group name, overrides LogGroupName + if set. + type: string + logKey: + description: If set, only the value of the key will be sent to + CloudWatch + type: string + logRetentionDays: + description: Number of days logs are retained for + enum: + - 1 + - 3 + - 5 + - 7 + - 14 + - 30 + - 60 + - 90 + - 120 + - 150 + - 180 + - 365 + - 400 + - 545 + - 731 + - 1827 + - 3653 + format: int32 + type: integer + logStreamName: + description: The name of the CloudWatch Log Stream to send log + records to + type: string + logStreamPrefix: + description: Prefix for the Log Stream name. Not compatible with + LogStreamName setting + type: string + logStreamTemplate: + description: Template for Log Stream name. Overrides LogStreamPrefix + and LogStreamName if set. + type: string + metricDimensions: + description: Optional lists of lists for dimension keys to be + added to all metrics. Use comma separated strings for one list + of dimensions and semicolon separated strings for list of lists + dimensions. + type: string + metricNamespace: + description: Optional string to represent the CloudWatch namespace. + type: string + region: + description: AWS Region + type: string + roleArn: + description: Role ARN to use for cross-account access + type: string + stsEndpoint: + description: Specify a custom STS endpoint for the AWS STS API + type: string + required: + - region + type: object customPlugin: description: CustomPlugin defines Custom Output configuration. properties: diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 7edf4de96..55a89c18b 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -2179,6 +2179,93 @@ spec: - customerID - sharedKey type: object + cloudWatch: + description: CloudWatch defines CloudWatch Output Configuration + properties: + autoCreateGroup: + description: Automatically create the log group. Defaults to False. + type: boolean + autoRetryRequests: + description: Automatically retry failed requests to CloudWatch + once. Defaults to True. + type: boolean + endpoint: + description: Custom endpoint for CloudWatch logs API + type: string + externalID: + description: Specify an external ID for the STS API. + type: string + logFormat: + description: Optional parameter to tell CloudWatch the format + of the data + type: string + logGroupName: + description: Name of Cloudwatch Log Group to send log records + to + type: string + logGroupTemplate: + description: Template for Log Group name, overrides LogGroupName + if set. + type: string + logKey: + description: If set, only the value of the key will be sent to + CloudWatch + type: string + logRetentionDays: + description: Number of days logs are retained for + enum: + - 1 + - 3 + - 5 + - 7 + - 14 + - 30 + - 60 + - 90 + - 120 + - 150 + - 180 + - 365 + - 400 + - 545 + - 731 + - 1827 + - 3653 + format: int32 + type: integer + logStreamName: + description: The name of the CloudWatch Log Stream to send log + records to + type: string + logStreamPrefix: + description: Prefix for the Log Stream name. Not compatible with + LogStreamName setting + type: string + logStreamTemplate: + description: Template for Log Stream name. Overrides LogStreamPrefix + and LogStreamName if set. + type: string + metricDimensions: + description: Optional lists of lists for dimension keys to be + added to all metrics. Use comma separated strings for one list + of dimensions and semicolon separated strings for list of lists + dimensions. + type: string + metricNamespace: + description: Optional string to represent the CloudWatch namespace. + type: string + region: + description: AWS Region + type: string + roleArn: + description: Role ARN to use for cross-account access + type: string + stsEndpoint: + description: Specify a custom STS endpoint for the AWS STS API + type: string + required: + - region + type: object customPlugin: description: CustomPlugin defines Custom Output configuration. properties: