Skip to content

Add log to metrics plugin #1305

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 20, 2024
Merged

Add log to metrics plugin #1305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a644b4
Add map for log-to-metrics plugin
Athishpranav2003 Aug 15, 2024
38a2605
Run make test and docs for autogen files
Athishpranav2003 Aug 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apis/fluentbit/v1alpha2/clusterfilter_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type FilterItem struct {
AWS *filter.AWS `json:"aws,omitempty"`
// Multiline defines a Multiline configuration.
Multiline *filter.Multiline `json:"multiline,omitempty"`
// LogToMetrics defines a Log to Metrics Filter configuration.
LogToMetrics *filter.LogToMetrics `json:"logToMetrics,omitempty"`
// CustomPlugin defines a Custom plugin configuration.
CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"`
}
Expand Down
112 changes: 112 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package filter

import (
"fmt"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// The Log To Metrics Filter plugin allows you to generate log-derived metrics. <br />
// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/log_to_metrics**
type LogToMetrics struct {
plugins.CommonParams `json:",inline"`
// Defines the tag for the generated metrics record
Tag string `json:"tag,omitempty"`
// Optional filter for records in which the content of KEY matches the regular expression.
// Value Format: FIELD REGEX
Regex []string `json:"regex,omitempty"`
// Optional filter for records in which the content of KEY does not matches the regular expression.
// Value Format: FIELD REGEX
Exclude []string `json:"exclude,omitempty"`
// Defines the mode for the metric. Valid values are [counter, gauge or histogram]
MetricMode string `json:"metricMode,omitempty"`
// Sets the name of the metric.
MetricName string `json:"metricName,omitempty"`
// Namespace of the metric
MetricNamespace string `json:"metricNamespace,omitempty"`
// Sets a sub-system for the metric.
MetricSubsystem string `json:"metricSubsystem,omitempty"`
// Sets a help text for the metric.
MetricDescription string `json:"metricDescription,omitempty"`
// Defines a bucket for histogram
Bucket []string `json:"bucket,omitempty"`
// Add a custom label NAME and set the value to the value of KEY
AddLabel []string `json:"addLabel,omitempty"`
// Includes a record field as label dimension in the metric.
LabelField []string `json:"labelField,omitempty"`
// Specify the record field that holds a numerical value
ValueField string `json:"valueField,omitempty"`
// If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name
// into the metric as labels. This option is intended to be used in combination with the kubernetes filter plugin.
KubernetesMode *bool `json:"kubernetesMode,omitempty"`
// Name of the emitter (advanced users)
EmitterName string `json:"emitterName,omitempty"`
// set a buffer limit to restrict memory usage of metrics emitter
EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"`
// Flag that defines if logs should be discarded after processing. This applies
// for all logs, no matter if they have emitted metrics or not.
DiscardLogs *bool `json:"discardLogs,omitempty"`
}

func (_ *LogToMetrics) Name() string {
return "log_to_metrics"
}

func (l *LogToMetrics) Params(_ plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()
err := l.AddCommonParams(kvs)
if err != nil {
return kvs, err
}
if l.Tag != "" {
kvs.Insert("Tag", l.Tag)
}
for _, reg := range l.Regex {
kvs.Insert("Regex", reg)
}
for _, ex := range l.Exclude {
kvs.Insert("Exclude", ex)
}
if l.MetricMode != "" {
kvs.Insert("Metric_mode", l.MetricMode)
}
if l.MetricName != "" {
kvs.Insert("Metric_name", l.MetricName)
}
if l.MetricNamespace != "" {
kvs.Insert("Metric_namespace", l.MetricNamespace)
}
if l.MetricSubsystem != "" {
kvs.Insert("Metric_subsystem", l.MetricSubsystem)
}
if l.MetricDescription != "" {
kvs.Insert("Metric_description", l.MetricDescription)
}
for _, b := range l.Bucket {
kvs.Insert("Bucket", b)
}
for _, al := range l.AddLabel {
kvs.Insert("Add_label", al)
}
for _, lf := range l.LabelField {
kvs.Insert("Label_field", lf)
}
if l.ValueField != "" {
kvs.Insert("Value_field", l.ValueField)
}
if l.KubernetesMode != nil {
kvs.Insert("Kubernetes_mode", fmt.Sprintf("%t", *l.KubernetesMode))
}
if l.EmitterName != "" {
kvs.Insert("Emitter_Name", l.EmitterName)
}
if l.EmitterMemBufLimit != "" {
kvs.Insert("Emitter_Mem_Buf_Limit", l.EmitterMemBufLimit)
}
if l.DiscardLogs != nil {
kvs.Insert("Discard_logs", fmt.Sprintf("%t", *l.DiscardLogs))
}
return kvs, nil
}
51 changes: 51 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions apis/fluentbit/v1alpha2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

88 changes: 88 additions & 0 deletions charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,94 @@ spec:
This could mitigate the Kube API heavy traffic issue for large cluster.
type: boolean
type: object
logToMetrics:
description: LogToMetrics defines a Log to Metrics Filter configuration.
properties:
addLabel:
description: Add a custom label NAME and set the value to
the value of KEY
items:
type: string
type: array
alias:
description: Alias for the plugin
type: string
bucket:
description: Defines a bucket for histogram
items:
type: string
type: array
discardLogs:
description: |-
Flag that defines if logs should be discarded after processing. This applies
for all logs, no matter if they have emitted metrics or not.
type: boolean
emitterMemBufLimit:
description: set a buffer limit to restrict memory usage
of metrics emitter
type: string
emitterName:
description: Name of the emitter (advanced users)
type: string
exclude:
description: |-
Optional filter for records in which the content of KEY does not matches the regular expression.
Value Format: FIELD REGEX
items:
type: string
type: array
kubernetesMode:
description: |-
If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name
into the metric as labels. This option is intended to be used in combination with the kubernetes filter plugin.
type: boolean
labelField:
description: Includes a record field as label dimension
in the metric.
items:
type: string
type: array
metricDescription:
description: Sets a help text for the metric.
type: string
metricMode:
description: Defines the mode for the metric. Valid values
are [counter, gauge or histogram]
type: string
metricName:
description: Sets the name of the metric.
type: string
metricNamespace:
description: Namespace of the metric
type: string
metricSubsystem:
description: Sets a sub-system for the metric.
type: string
regex:
description: |-
Optional filter for records in which the content of KEY matches the regular expression.
Value Format: FIELD REGEX
items:
type: string
type: array
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
to false fluent-bit will try indefinetly. If set to any
integer N>0 it will try at most N+1 times. Leading zeros
are not allowed (values such as 007, 0150, 01 do not work).
If this property is not defined fluent-bit will use the
default value: 1.'
pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
type: string
tag:
description: Defines the tag for the generated metrics record
type: string
valueField:
description: Specify the record field that holds a numerical
value
type: string
type: object
lua:
description: Lua defines Lua Filter configuration.
properties:
Expand Down
88 changes: 88 additions & 0 deletions charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,94 @@ spec:
This could mitigate the Kube API heavy traffic issue for large cluster.
type: boolean
type: object
logToMetrics:
description: LogToMetrics defines a Log to Metrics Filter configuration.
properties:
addLabel:
description: Add a custom label NAME and set the value to
the value of KEY
items:
type: string
type: array
alias:
description: Alias for the plugin
type: string
bucket:
description: Defines a bucket for histogram
items:
type: string
type: array
discardLogs:
description: |-
Flag that defines if logs should be discarded after processing. This applies
for all logs, no matter if they have emitted metrics or not.
type: boolean
emitterMemBufLimit:
description: set a buffer limit to restrict memory usage
of metrics emitter
type: string
emitterName:
description: Name of the emitter (advanced users)
type: string
exclude:
description: |-
Optional filter for records in which the content of KEY does not matches the regular expression.
Value Format: FIELD REGEX
items:
type: string
type: array
kubernetesMode:
description: |-
If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name
into the metric as labels. This option is intended to be used in combination with the kubernetes filter plugin.
type: boolean
labelField:
description: Includes a record field as label dimension
in the metric.
items:
type: string
type: array
metricDescription:
description: Sets a help text for the metric.
type: string
metricMode:
description: Defines the mode for the metric. Valid values
are [counter, gauge or histogram]
type: string
metricName:
description: Sets the name of the metric.
type: string
metricNamespace:
description: Namespace of the metric
type: string
metricSubsystem:
description: Sets a sub-system for the metric.
type: string
regex:
description: |-
Optional filter for records in which the content of KEY matches the regular expression.
Value Format: FIELD REGEX
items:
type: string
type: array
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
to false fluent-bit will try indefinetly. If set to any
integer N>0 it will try at most N+1 times. Leading zeros
are not allowed (values such as 007, 0150, 01 do not work).
If this property is not defined fluent-bit will use the
default value: 1.'
pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
type: string
tag:
description: Defines the tag for the generated metrics record
type: string
valueField:
description: Specify the record field that holds a numerical
value
type: string
type: object
lua:
description: Lua defines Lua Filter configuration.
properties:
Expand Down
Loading