Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for aws filter plugin #173

Merged
merged 1 commit into from
Nov 27, 2021
Merged

Add support for aws filter plugin #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ The list below shows supported plugins which are based on Fluent Bit v1.7.x+. Fo
- [record modifier](docs/plugins/filter/recordmodifier.md)
- [lua](docs/plugins/filter/lua.md)
- [throttle](docs/plugins/filter/throttle.md)
- [aws](docs/plugins/filter/aws.md)
- [Output](docs/crd.md#output)
- [elasticsearch](docs/plugins/output/elasticsearch.md)
- [file](docs/plugins/output/file.md)
Expand Down
2 changes: 2 additions & 0 deletions api/fluentbitoperator/v1alpha2/filter_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ type FilterItem struct {
Throttle *filter.Throttle `json:"throttle,omitempty"`
// RewriteTag defines a RewriteTag configuration.
RewriteTag *filter.RewriteTag `json:"rewriteTag,omitempty"`
//Aws defines a Aws configuration.
AWS *filter.AWS `json:"aws,omitempty"`
}

// +kubebuilder:object:root=true
Expand Down
69 changes: 69 additions & 0 deletions api/fluentbitoperator/v1alpha2/plugins/filter/aws_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package filter

import (
"fmt"

"kubesphere.io/fluentbit-operator/api/fluentbitoperator/v1alpha2/plugins"
"kubesphere.io/fluentbit-operator/api/fluentbitoperator/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// The AWS Filter Enriches logs with AWS Metadata. Currently the plugin adds the EC2 instance ID and availability zone to log records.
type AWS struct {
// Specify which version of the instance metadata service to use. Valid values are 'v1' or 'v2'.
// +kubebuilder:validation:Enum:=v1;v2
ImdsVersion string `json:"imdsVersion,omitempty"`
// The availability zone; for example, "us-east-1a". Default is true.
AZ *bool `json:"az,omitempty"`
//The EC2 instance ID.Default is true.
EC2InstanceID *bool `json:"ec2InstanceID,omitempty"`
//The EC2 instance type.Default is false.
EC2InstanceType *bool `json:"ec2InstanceType,omitempty"`
//The EC2 instance private ip.Default is false.
PrivateIP *bool `json:"privateIP,omitempty"`
//The EC2 instance image id.Default is false.
AmiID *bool `json:"amiID,omitempty"`
//The account ID for current EC2 instance.Default is false.
AccountID *bool `json:"accountID,omitempty"`
//The hostname for current EC2 instance.Default is false.
HostName *bool `json:"hostName,omitempty"`
//The VPC ID for current EC2 instance.Default is false.
VpcID *bool `json:"vpcID,omitempty"`
}

func (_ *AWS) Name() string {
return "aws"
}

func (a *AWS) Params(_ plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()
if a.ImdsVersion != "" {
kvs.Insert("imds_version", a.ImdsVersion)
}
if a.AZ != nil {
kvs.Insert("az", fmt.Sprint(*a.AZ))
}
if a.EC2InstanceID != nil {
kvs.Insert("ec2_instance_id", fmt.Sprint(*a.EC2InstanceID))
}
if a.EC2InstanceType != nil {
kvs.Insert("ec2_instance_type", fmt.Sprint(*a.EC2InstanceType))
}
if a.PrivateIP != nil {
kvs.Insert("private_ip", fmt.Sprint(*a.PrivateIP))
}
if a.AmiID != nil {
kvs.Insert("ami_id", fmt.Sprint(*a.AmiID))
}
if a.AccountID != nil {
kvs.Insert("account_id", fmt.Sprint(*a.AccountID))
}
if a.HostName != nil {
kvs.Insert("hostname", fmt.Sprint(*a.HostName))
}
if a.VpcID != nil {
kvs.Insert("vpc_id", fmt.Sprint(*a.VpcID))
}
return kvs, nil
}
55 changes: 55 additions & 0 deletions api/fluentbitoperator/v1alpha2/plugins/filter/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions api/fluentbitoperator/v1alpha2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 39 additions & 0 deletions config/crd/bases/logging.kubesphere.io_filters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,45 @@ spec:
description: A set of filter plugins in order.
items:
properties:
aws:
description: Aws defines a Aws configuration.
properties:
accountID:
description: The account ID for current EC2 instance.Default
is false.
type: boolean
amiID:
description: The EC2 instance image id.Default is false.
type: boolean
az:
description: The availability zone; for example, "us-east-1a".
Default is true.
type: boolean
ec2InstanceID:
description: The EC2 instance ID.Default is true.
type: boolean
ec2InstanceType:
description: The EC2 instance type.Default is false.
type: boolean
hostName:
description: The hostname for current EC2 instance.Default
is false.
type: boolean
imdsVersion:
description: Specify which version of the instance metadata
service to use. Valid values are 'v1' or 'v2'.
enum:
- v1
- v2
type: string
privateIP:
description: The EC2 instance private ip.Default is false.
type: boolean
vpcID:
description: The VPC ID for current EC2 instance.Default
is false.
type: boolean
type: object
grep:
description: Grep defines Grep Filter configuration.
properties:
Expand Down
19 changes: 19 additions & 0 deletions docs/plugins/filter/aws.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# AWS Metadata

The AWS Filter Enriches logs with AWS Metadata. Currently the plugin adds the EC2 instance ID and availability zone to log records. To use this plugin, you must be running in EC2 and have the [instance metadata service enabled](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html).


| Field | Description | Scheme |Default|
| ----- | ----------- | ------ | -----|
| imds_version | Specify which version of the instance metadata service to use. Valid values are 'v1' or 'v2'. | string |v2|
| az | The availability zone; for example, "us-east-1a". | bool |true|
| ec2_instance_id | The EC2 instance ID. | bool |true|
| ec2_instance_type | The EC2 instance type. | bool |false|
| private_ip | The EC2 instance private ip.| bool |false|
| ami_id | The EC2 instance image id. | bool |false|
| account_id | The account ID for current EC2 instance. | bool |false|
| hostname | The hostname for current EC2 instance.| bool |false|
| vpc_id | The VPC ID for current EC2 instance. | bool |false|


Note: If you run Fluent Bit in a container, you may have to use instance metadata v1. The plugin behaves the same regardless of which version is used.
39 changes: 39 additions & 0 deletions manifests/setup/fluentbit-operator-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,45 @@ spec:
description: A set of filter plugins in order.
items:
properties:
aws:
description: Aws defines a Aws configuration.
properties:
accountID:
description: The account ID for current EC2 instance.Default
is false.
type: boolean
amiID:
description: The EC2 instance image id.Default is false.
type: boolean
az:
description: The availability zone; for example, "us-east-1a".
Default is true.
type: boolean
ec2InstanceID:
description: The EC2 instance ID.Default is true.
type: boolean
ec2InstanceType:
description: The EC2 instance type.Default is false.
type: boolean
hostName:
description: The hostname for current EC2 instance.Default
is false.
type: boolean
imdsVersion:
description: Specify which version of the instance metadata
service to use. Valid values are 'v1' or 'v2'.
enum:
- v1
- v2
type: string
privateIP:
description: The EC2 instance private ip.Default is false.
type: boolean
vpcID:
description: The VPC ID for current EC2 instance.Default
is false.
type: boolean
type: object
grep:
description: Grep defines Grep Filter configuration.
properties:
Expand Down
39 changes: 39 additions & 0 deletions manifests/setup/setup.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,45 @@ spec:
description: A set of filter plugins in order.
items:
properties:
aws:
description: Aws defines a Aws configuration.
properties:
accountID:
description: The account ID for current EC2 instance.Default
is false.
type: boolean
amiID:
description: The EC2 instance image id.Default is false.
type: boolean
az:
description: The availability zone; for example, "us-east-1a".
Default is true.
type: boolean
ec2InstanceID:
description: The EC2 instance ID.Default is true.
type: boolean
ec2InstanceType:
description: The EC2 instance type.Default is false.
type: boolean
hostName:
description: The hostname for current EC2 instance.Default
is false.
type: boolean
imdsVersion:
description: Specify which version of the instance metadata
service to use. Valid values are 'v1' or 'v2'.
enum:
- v1
- v2
type: string
privateIP:
description: The EC2 instance private ip.Default is false.
type: boolean
vpcID:
description: The VPC ID for current EC2 instance.Default
is false.
type: boolean
type: object
grep:
description: Grep defines Grep Filter configuration.
properties:
Expand Down