Add support for prometheus expoter output plugin for fluentbit

apis/fluentbit/v1alpha2/clusteroutput_types.go

@@ -92,6 +92,8 @@ type OutputSpec struct {
 	OpenSearch *output.OpenSearch `json:"opensearch,omitempty"`
 	// OpenTelemetry defines OpenTelemetry Output configuration.
 	OpenTelemetry *output.OpenTelemetry `json:"opentelemetry,omitempty"`
+	// PrometheusExporter_types defines Prometheus exporter configuration to expose metrics from Fluent Bit.
+	PrometheusExporter *output.PrometheusExporter `json:"prometheusExporter,omitempty"`
 	// PrometheusRemoteWrite_types defines Prometheus Remote Write configuration.
 	PrometheusRemoteWrite *output.PrometheusRemoteWrite `json:"prometheusRemoteWrite,omitempty"`
 	// S3 defines S3 Output configuration.

apis/fluentbit/v1alpha2/plugins/output/prometheus_exporter.go

@@ -0,0 +1,45 @@
+package output
+
+import (
+	"fmt"
+
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
+)
+
+// +kubebuilder:object:generate:=true
+
+// PrometheusExporter An output plugin to expose Prometheus Metrics. <br />
+// The prometheus exporter allows you to take metrics from Fluent Bit and expose them such that a Prometheus instance can scrape them. <br />
+// **Important Note: The prometheus exporter only works with metric plugins, such as Node Exporter Metrics** <br />
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/prometheus-exporter**
+type PrometheusExporter struct {
+	// IP address or hostname of the target HTTP Server, default: 0.0.0.0
+	Host string `json:"host"`
+	// This is the port Fluent Bit will bind to when hosting prometheus metrics.
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=65535
+	Port *int32 `json:"port,omitempty"`
+	//This allows you to add custom labels to all metrics exposed through the prometheus exporter. You may have multiple of these fields
+	AddLabels map[string]string `json:"addLabels,omitempty"`
+}
+
+// implement Section() method
+func (_ *PrometheusExporter) Name() string {
+	return "prometheus_exporter"
+}
+
+// implement Section() method
+func (p *PrometheusExporter) Params(sl plugins.SecretLoader) (*params.KVs, error) {
+	kvs := params.NewKVs()
+	if p.Host != "" {
+		kvs.Insert("host", p.Host)
+	}
+	if p.Port != nil {
+		kvs.Insert("port", fmt.Sprint(*p.Port))
+	}
+	kvs.InsertStringMap(p.AddLabels, func(k, v string) (string, string) {
+		return "add_label", fmt.Sprintf(" %s    %s", k, v)
+	})
+	return kvs, nil
+}

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml

@@ -2038,6 +2038,31 @@ spec:
                       server listening for traces, e.g: /v1/traces'
                     type: string
                 type: object
+              prometheusExporter:
+                description: PrometheusExporter_types defines Prometheus exporter
+                  configuration to expose metrics from Fluent Bit.
+                properties:
+                  addLabels:
+                    additionalProperties:
+                      type: string
+                    description: This allows you to add custom labels to all metrics
+                      exposed through the prometheus exporter. You may have multiple
+                      of these fields
+                    type: object
+                  host:
+                    description: 'IP address or hostname of the target HTTP Server,
+                      default: 0.0.0.0'
+                    type: string
+                  port:
+                    description: This is the port Fluent Bit will bind to when hosting
+                      prometheus metrics.
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                required:
+                - host
+                type: object
               prometheusRemoteWrite:
                 description: PrometheusRemoteWrite_types defines Prometheus Remote
                   Write configuration.

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml

@@ -2038,6 +2038,31 @@ spec:
                       server listening for traces, e.g: /v1/traces'
                     type: string
                 type: object
+              prometheusExporter:
+                description: PrometheusExporter_types defines Prometheus exporter
+                  configuration to expose metrics from Fluent Bit.
+                properties:
+                  addLabels:
+                    additionalProperties:
+                      type: string
+                    description: This allows you to add custom labels to all metrics
+                      exposed through the prometheus exporter. You may have multiple
+                      of these fields
+                    type: object
+                  host:
+                    description: 'IP address or hostname of the target HTTP Server,
+                      default: 0.0.0.0'
+                    type: string
+                  port:
+                    description: This is the port Fluent Bit will bind to when hosting
+                      prometheus metrics.
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                required:
+                - host
+                type: object
               prometheusRemoteWrite:
                 description: PrometheusRemoteWrite_types defines Prometheus Remote
                   Write configuration.

charts/fluent-operator/templates/fluentbit-clusterinput-metrics.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.fluentbit.enable -}}
+{{- if .Values.fluentbit.input.fluentBitMetrics -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterInput
+metadata:
+  name: fluentbit-metrics
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  fluentBitMetrics:
+{{ toYaml .Values.fluentbit.input.fluentBitMetrics | indent 4}}
+{{- end }}
+{{- end }}

charts/fluent-operator/templates/fluentbit-output-prometheus-exporter.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.fluentbit.enable -}}
+{{- if .Values.fluentbit.output.prometheusMetricsExporter -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterOutput
+metadata:
+  name: prometheus-exporter
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  match: {{ .Values.fluentbit.output.prometheusMetricsExporter.match }}
+  prometheusExporter:
+{{ toYaml .Values.fluentbit.output.prometheusMetricsExporter.metricsExporter | indent 4}}
+{{- end }}
+{{- end }}

charts/fluent-operator/values.yaml

@@ -191,6 +191,12 @@ fluentbit:
   #     path:
   #       procfs: /host/proc
   #       sysfs: /host/sys
+    fluentBitMetrics: {}
+    # uncomment below fluentBitMetrics section if you want to collect fluentBit metrics
+#    fluentBitMetrics:
+#      scrapeInterval: "2"
+#      scrapeOnStart: true
+#      tag: "fb.metrics"
 
   # Configure the output plugin parameter in FluentBit.
   # You can set enable to true to output logs to the specified location.
@@ -228,6 +234,15 @@ fluentbit:
     # You can configure the opensearch-related configuration here
     stdout:
       enable: false
+    # Uncomment the following section to enable Prometheus metrics exporter.
+    prometheusMetricsExporter: {}
+#    prometheusMetricsExporter:
+#      match: "fb.metrics"
+#      metricsExporter:
+#        host: "0.0.0.0"
+#        port: 2020
+#        addLabels:
+#          app: "fluentbit"
   service:
     storage: {}
 # Remove the above storage section and uncomment below section if you want to configure file-system as storage for buffer

config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml

@@ -2038,6 +2038,31 @@ spec:
                       server listening for traces, e.g: /v1/traces'
                     type: string
                 type: object
+              prometheusExporter:
+                description: PrometheusExporter_types defines Prometheus exporter
+                  configuration to expose metrics from Fluent Bit.
+                properties:
+                  addLabels:
+                    additionalProperties:
+                      type: string
+                    description: This allows you to add custom labels to all metrics
+                      exposed through the prometheus exporter. You may have multiple
+                      of these fields
+                    type: object
+                  host:
+                    description: 'IP address or hostname of the target HTTP Server,
+                      default: 0.0.0.0'
+                    type: string
+                  port:
+                    description: This is the port Fluent Bit will bind to when hosting
+                      prometheus metrics.
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                required:
+                - host
+                type: object
               prometheusRemoteWrite:
                 description: PrometheusRemoteWrite_types defines Prometheus Remote
                   Write configuration.

config/crd/bases/fluentbit.fluent.io_outputs.yaml

@@ -2038,6 +2038,31 @@ spec:
                       server listening for traces, e.g: /v1/traces'
                     type: string
                 type: object
+              prometheusExporter:
+                description: PrometheusExporter_types defines Prometheus exporter
+                  configuration to expose metrics from Fluent Bit.
+                properties:
+                  addLabels:
+                    additionalProperties:
+                      type: string
+                    description: This allows you to add custom labels to all metrics
+                      exposed through the prometheus exporter. You may have multiple
+                      of these fields
+                    type: object
+                  host:
+                    description: 'IP address or hostname of the target HTTP Server,
+                      default: 0.0.0.0'
+                    type: string
+                  port:
+                    description: This is the port Fluent Bit will bind to when hosting
+                      prometheus metrics.
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                required:
+                - host
+                type: object
               prometheusRemoteWrite:
                 description: PrometheusRemoteWrite_types defines Prometheus Remote
                   Write configuration.

go.mod

@@ -58,12 +58,14 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
@@ -72,7 +74,9 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/code-generator v0.26.1 // indirect
 	k8s.io/component-base v0.26.1 // indirect
+	k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect