Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding Cloudwatch for Fluentbit Output Plugin #609

Merged
merged 1 commit into from
Mar 10, 2023
Merged

Adding Cloudwatch for Fluentbit Output Plugin #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions apis/fluentbit/v1alpha2/clusteroutput_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ type OutputSpec struct {
AzureBlob *output.AzureBlob `json:"azureBlob,omitempty"`
// AzureLogAnalytics defines AzureLogAnalytics Output Configuration
AzureLogAnalytics *output.AzureLogAnalytics `json:"azureLogAnalytics,omitempty"`
// CloudWatch defines CloudWatch Output Configuration
CloudWatch *output.CloudWatch `json:"cloudWatch,omitempty"`
// RetryLimit represents configuration for the scheduler which can be set independently on each output section.
// This option allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit.
RetryLimit string `json:"retry_limit,omitempty"`
Expand Down
111 changes: 111 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/output/cloudwatch_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package output

import (
"fmt"
"github.com/fluent/fluent-operator/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/apis/fluentbit/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// CloudWatch is the AWS CloudWatch output plugin, allows you to ingest your records into AWS CloudWatch.
type CloudWatch struct {
// AWS Region
Region string `json:"region"`
// Name of Cloudwatch Log Group to send log records to
LogGroupName string `json:"logGroupName,omitempty"`
// Template for Log Group name, overrides LogGroupName if set.
LogGroupTemplate string `json:"logGroupTemplate,omitempty"`
// The name of the CloudWatch Log Stream to send log records to
LogStreamName string `json:"logStreamName,omitempty"`
// Prefix for the Log Stream name. Not compatible with LogStreamName setting
LogStreamPrefix string `json:"logStreamPrefix,omitempty"`
// Template for Log Stream name. Overrides LogStreamPrefix and LogStreamName if set.
LogStreamTemplate string `json:"logStreamTemplate,omitempty"`
// If set, only the value of the key will be sent to CloudWatch
LogKey string `json:"logKey,omitempty"`
// Optional parameter to tell CloudWatch the format of the data
LogFormat string `json:"logFormat,omitempty"`
// Role ARN to use for cross-account access
RoleArn string `json:"roleArn,omitempty"`
// Automatically create the log group. Defaults to False.
AutoCreateGroup *bool `json:"autoCreateGroup,omitempty"`
// Number of days logs are retained for
// +kubebuilder:validation:Enum:=1;3;5;7;14;30;60;90;120;150;180;365;400;545;731;1827;3653
LogRetentionDays *int32 `json:"logRetentionDays,omitempty"`
// Custom endpoint for CloudWatch logs API
Endpoint string `json:"endpoint,omitempty"`
// Optional string to represent the CloudWatch namespace.
MetricNamespace string `json:"metricNamespace,omitempty"`
// Optional lists of lists for dimension keys to be added to all metrics. Use comma separated strings
// for one list of dimensions and semicolon separated strings for list of lists dimensions.
MetricDimensions string `json:"metricDimensions,omitempty"`
// Specify a custom STS endpoint for the AWS STS API
StsEndpoint string `json:"stsEndpoint,omitempty"`
// Automatically retry failed requests to CloudWatch once. Defaults to True.
AutoRetryRequests *bool `json:"autoRetryRequests,omitempty"`
// Specify an external ID for the STS API.
ExternalID string `json:"externalID,omitempty"`
}

// Name implement Section() method
func (_ *CloudWatch) Name() string {
return "cloudwatch_logs"
}

// Params implement Section() method
func (o *CloudWatch) Params(sl plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()
if o.Region != "" {
kvs.Insert("region", o.Region)
}
if o.LogGroupName != "" {
kvs.Insert("log_group_name", o.LogGroupName)
}
if o.LogGroupTemplate != "" {
kvs.Insert("log_group_template", o.LogGroupTemplate)
}
if o.LogStreamName != "" {
kvs.Insert("log_stream_name", o.LogStreamName)
}
if o.LogStreamPrefix != "" {
kvs.Insert("log_stream_prefix", o.LogStreamPrefix)
}
if o.LogStreamTemplate != "" {
kvs.Insert("log_stream_template", o.LogStreamTemplate)
}
if o.LogKey != "" {
kvs.Insert("log_key", o.LogKey)
}
if o.LogFormat != "" {
kvs.Insert("log_format", o.LogFormat)
}
if o.AutoCreateGroup != nil {
kvs.Insert("auto_create_group", fmt.Sprint(*o.AutoCreateGroup))
}
if o.LogRetentionDays != nil {
kvs.Insert("log_retention_days", fmt.Sprint(*o.LogRetentionDays))
}
if o.RoleArn != "" {
kvs.Insert("role_arn", o.RoleArn)
}
if o.Endpoint != "" {
kvs.Insert("endpoint", o.Endpoint)
}
if o.MetricNamespace != "" {
kvs.Insert("metric_namespace", o.MetricNamespace)
}
if o.MetricDimensions != "" {
kvs.Insert("metric_dimensions", o.MetricDimensions)
}
if o.StsEndpoint != "" {
kvs.Insert("sts_endpoint", o.StsEndpoint)
}
if o.AutoRetryRequests != nil {
kvs.Insert("auto_retry_requests", fmt.Sprint(*o.AutoRetryRequests))
}
if o.ExternalID != "" {
kvs.Insert("external_id", o.ExternalID)
}
return kvs, nil
}
87 changes: 87 additions & 0 deletions charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,93 @@ spec:
- customerID
- sharedKey
type: object
cloudWatch:
description: CloudWatch defines CloudWatch Output Configuration
properties:
autoCreateGroup:
description: Automatically create the log group. Defaults to False.
type: boolean
autoRetryRequests:
description: Automatically retry failed requests to CloudWatch
once. Defaults to True.
type: boolean
endpoint:
description: Custom endpoint for CloudWatch logs API
type: string
externalID:
description: Specify an external ID for the STS API.
type: string
logFormat:
description: Optional parameter to tell CloudWatch the format
of the data
type: string
logGroupName:
description: Name of Cloudwatch Log Group to send log records
to
type: string
logGroupTemplate:
description: Template for Log Group name, overrides LogGroupName
if set.
type: string
logKey:
description: If set, only the value of the key will be sent to
CloudWatch
type: string
logRetentionDays:
description: Number of days logs are retained for
enum:
- 1
- 3
- 5
- 7
- 14
- 30
- 60
- 90
- 120
- 150
- 180
- 365
- 400
- 545
- 731
- 1827
- 3653
format: int32
type: integer
logStreamName:
description: The name of the CloudWatch Log Stream to send log
records to
type: string
logStreamPrefix:
description: Prefix for the Log Stream name. Not compatible with
LogStreamName setting
type: string
logStreamTemplate:
description: Template for Log Stream name. Overrides LogStreamPrefix
and LogStreamName if set.
type: string
metricDimensions:
description: Optional lists of lists for dimension keys to be
added to all metrics. Use comma separated strings for one list
of dimensions and semicolon separated strings for list of lists
dimensions.
type: string
metricNamespace:
description: Optional string to represent the CloudWatch namespace.
type: string
region:
description: AWS Region
type: string
roleArn:
description: Role ARN to use for cross-account access
type: string
stsEndpoint:
description: Specify a custom STS endpoint for the AWS STS API
type: string
required:
- region
type: object
customPlugin:
description: CustomPlugin defines Custom Output configuration.
properties:
Expand Down
87 changes: 87 additions & 0 deletions config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,93 @@ spec:
- customerID
- sharedKey
type: object
cloudWatch:
description: CloudWatch defines CloudWatch Output Configuration
properties:
autoCreateGroup:
description: Automatically create the log group. Defaults to False.
type: boolean
autoRetryRequests:
description: Automatically retry failed requests to CloudWatch
once. Defaults to True.
type: boolean
endpoint:
description: Custom endpoint for CloudWatch logs API
type: string
externalID:
description: Specify an external ID for the STS API.
type: string
logFormat:
description: Optional parameter to tell CloudWatch the format
of the data
type: string
logGroupName:
description: Name of Cloudwatch Log Group to send log records
to
type: string
logGroupTemplate:
description: Template for Log Group name, overrides LogGroupName
if set.
type: string
logKey:
description: If set, only the value of the key will be sent to
CloudWatch
type: string
logRetentionDays:
description: Number of days logs are retained for
enum:
- 1
- 3
- 5
- 7
- 14
- 30
- 60
- 90
- 120
- 150
- 180
- 365
- 400
- 545
- 731
- 1827
- 3653
format: int32
type: integer
logStreamName:
description: The name of the CloudWatch Log Stream to send log
records to
type: string
logStreamPrefix:
description: Prefix for the Log Stream name. Not compatible with
LogStreamName setting
type: string
logStreamTemplate:
description: Template for Log Stream name. Overrides LogStreamPrefix
and LogStreamName if set.
type: string
metricDimensions:
description: Optional lists of lists for dimension keys to be
added to all metrics. Use comma separated strings for one list
of dimensions and semicolon separated strings for list of lists
dimensions.
type: string
metricNamespace:
description: Optional string to represent the CloudWatch namespace.
type: string
region:
description: AWS Region
type: string
roleArn:
description: Role ARN to use for cross-account access
type: string
stsEndpoint:
description: Specify a custom STS endpoint for the AWS STS API
type: string
required:
- region
type: object
customPlugin:
description: CustomPlugin defines Custom Output configuration.
properties:
Expand Down
1 change: 1 addition & 0 deletions docs/fluentbit.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,7 @@ OutputSpec defines the desired state of ClusterOutput
| alias | A user friendly alias name for this output plugin. Used in metrics for distinction of each configured output. | string |
| azureBlob | AzureBlob defines AzureBlob Output Configuration | *[output.AzureBlob](plugins/output/azureblob.md) |
| azureLogAnalytics | AzureLogAnalytics defines AzureLogAnalytics Output Configuration | *[output.AzureLogAnalytics](plugins/output/azureloganalytics.md) |
| cloudWatch | CloudWatch defines CloudWatch Output Configuration | *[output.CloudWatch](plugins/output/cloudwatch.md) |
| retry_limit | RetryLimit represents configuration for the scheduler which can be set independently on each output section. This option allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. | string |
| es | Elasticsearch defines Elasticsearch Output configuration. | *[output.Elasticsearch](plugins/output/elasticsearch.md) |
| file | File defines File Output configuration. | *[output.File](plugins/output/file.md) |
Expand Down