emitter type and buffers added

fluent · Feb 20, 2024 · be713dd · be713dd
1 parent 2f254cc
commit be713dd
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 0 deletions.
diff --git a/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go b/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go
@@ -22,6 +22,9 @@ type Multi struct {
 	//Key name that holds the content to process.
 	//Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter.
 	KeyContent string `json:"keyContent,omitempty"`
+	EmitterName string `json:"emitterName,omitempty"`
+	EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"`
+	EmitterStorageType string `json:"emitterStorageType,omitempty"`
 }
 
 func (_ *Multiline) Name() string {
@@ -41,6 +44,15 @@ func (m *Multiline) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 		if m.Multi.KeyContent != "" {
 			kvs.Insert("multiline.key_content", m.Multi.KeyContent)
 		}
+    	if m.Multi.EmitterName != "" {
+		    kvs.Insert("Emitter_Name", m.Multi.EmitterName)
+	    }
+	    if m.Multi.EmitterMemBufLimit != "" {
+		    kvs.Insert("Emitter_Mem_Buf_Limit", m.Multi.EmitterMemBufLimit)
+	    }
+	    if m.Multi.EmitterStorageType != "" {
+		    kvs.Insert("Emitter_Storage.type", m.Multi.EmitterStorageType)
+	    }
 	}
 	return kvs, nil
 }
diff --git a/apis/fluentbit/v1alpha2/plugins/filter/rewritetag_types.go b/apis/fluentbit/v1alpha2/plugins/filter/rewritetag_types.go
@@ -19,6 +19,8 @@ type RewriteTag struct {
 	// plugin that takes care of the job. Since this emitter expose metrics as any other
 	// component of the pipeline, you can use this property to configure an optional name for it.
 	EmitterName string `json:"emitterName,omitempty"`
+	EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"`
+	EmitterStorageType string `json:"emitterStorageType,omitempty"`
 }
 
 func (_ *RewriteTag) Name() string {
@@ -37,5 +39,11 @@ func (r *RewriteTag) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if r.EmitterName != "" {
 		kvs.Insert("Emitter_Name", r.EmitterName)
 	}
+	if r.EmitterMemBufLimit != "" {
+		kvs.Insert("Emitter_Mem_Buf_Limit", r.EmitterMemBufLimit)
+	}
+	if r.EmitterStorageType != "" {
+		kvs.Insert("Emitter_Storage.type", r.EmitterStorageType)
+	}
 	return kvs, nil
 }
diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml
@@ -486,6 +486,25 @@ spec:
                         alias:
                           description: Alias for the plugin
                           type: string
+                        emitterName:
+                          description: When the filter emits a record under the new
+                            Tag, there is an internal emitter plugin that takes care
+                            of the job. Since this emitter expose metrics as any other
+                            component of the pipeline, you can use this property to
+                            configure an optional name for it.
+                          type: string
+                        emitterStorageType:
+                          description: Specify the emitter buffering mechanism to use. It can be
+                            memory or filesystem
+                          enum:
+                          - filesystem
+                          - memory
+                          type: string
+                        emitterMemBufLimit:
+                          description: Set a limit of memory that Emitter plugin can use when
+                            appending data to the Engine. If the limit is reach, it will
+                            be paused; when the data is flushed it resumes.
+                          type: string
                         keyContent:
                           description: Key name that holds the content to process.
                             Note that a Multiline Parser definition can already specify
@@ -647,6 +666,18 @@ spec:
                             component of the pipeline, you can use this property to
                             configure an optional name for it.
                           type: string
+                        emitterStorageType:
+                          description: Specify the emitter buffering mechanism to use. It can be
+                            memory or filesystem
+                          enum:
+                          - filesystem
+                          - memory
+                          type: string
+                        emitterMemBufLimit:
+                          description: Set a limit of memory that Emitter plugin can use when
+                            appending data to the Engine. If the limit is reach, it will
+                            be paused; when the data is flushed it resumes.
+                          type: string
                         retryLimit:
                           description: 'RetryLimit describes how many times fluent-bit
                             should retry to send data to a specific output. If set