add multiline filter plugin

Signed-off-by: chengdehao <dehaocheng@yunify.com>

README.md

@@ -312,6 +312,7 @@ The list below shows supported plugins which are based on Fluent Bit v1.7.x+. Fo
   - [lua](docs/plugins/filter/lua.md)
   - [throttle](docs/plugins/filter/throttle.md)
   - [aws](docs/plugins/filter/aws.md)
+  - [multiline](docs/plugins/filter/multiline.md)
 - [Output](docs/crd.md#output)
   - [elasticsearch](docs/plugins/output/elasticsearch.md)
   - [file](docs/plugins/output/file.md)

api/fluentbitoperator/v1alpha2/filter_types.go

@@ -63,6 +63,8 @@ type FilterItem struct {
 	RewriteTag *filter.RewriteTag `json:"rewriteTag,omitempty"`
 	//Aws defines a Aws configuration.
 	AWS *filter.AWS `json:"aws,omitempty"`
+	//Multiline defines a Multiline configuration.
+	Multiline *filter.Multiline `json:"multiline,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/fluentbitoperator/v1alpha2/plugins/filter/multiline_types.go

@@ -0,0 +1,40 @@
+package filter
+
+import (
+	"kubesphere.io/fluentbit-operator/api/fluentbitoperator/v1alpha2/plugins"
+	"kubesphere.io/fluentbit-operator/api/fluentbitoperator/v1alpha2/plugins/params"
+)
+
+// +kubebuilder:object:generate:=true
+
+// The Multiline Filter helps to concatenate messages that originally belong to one context but were split across multiple records or log lines.
+type Multiline struct {
+	// The Inline struct helps to concatenate messages that originally belong to one context but were split across multiple records or log lines.
+	*Multi `json:",inline"`
+}
+
+type Multi struct {
+	// Specify one or multiple Multiline Parsing definitions to apply to the content.
+	//You can specify multiple multiline parsers to detect different formats by separating them with a comma.
+	Parser string `json:"parser"`
+	//Key name that holds the content to process.
+	//Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter.
+	KeyContent string `json:"keyContent,omitempty"`
+}
+
+func (_ *Multiline) Name() string {
+	return "multiline"
+}
+
+func (m *Multiline) Params(_ plugins.SecretLoader) (*params.KVs, error) {
+	kvs := params.NewKVs()
+	if m.Multi != nil {
+		if m.Multi.Parser != "" {
+			kvs.Insert("multiline.parser", m.Multi.Parser)
+		}
+		if m.Multi.KeyContent != "" {
+			kvs.Insert("multiline.key_content", m.Multi.KeyContent)
+		}
+	}
+	return kvs, nil
+}

config/crd/bases/logging.kubesphere.io_filters.yaml

@@ -364,6 +364,24 @@ spec:
                             type: object
                           type: array
                       type: object
+                    multiline:
+                      description: Multiline defines a Multiline configuration.
+                      properties:
+                        keyContent:
+                          description: Key name that holds the content to process.
+                            Note that a Multiline Parser definition can already specify
+                            the key_content to use, but this option allows to overwrite
+                            that value for the purpose of the filter.
+                          type: string
+                        parser:
+                          description: Specify one or multiple Multiline Parsing definitions
+                            to apply to the content. You can specify multiple multiline
+                            parsers to detect different formats by separating them
+                            with a comma.
+                          type: string
+                      required:
+                      - parser
+                      type: object
                     nest:
                       description: Nest defines Nest Filter configuration.
                       properties:

docs/plugins/filter/multiline.md

@@ -0,0 +1,9 @@
+# Multiline
+
+The Multiline Filter helps to concatenate messages that originally belong to one context but were split across multiple records or log lines. Common examples are stack traces or applications that print logs in multiple lines.
+
+
+| Field | Description | Scheme |Default|
+| ----- | ----------- | ------ | -----|
+| multiline.parser | Specify one or multiple Multiline Parser definitions to apply to the content. You can specify multiple multiline parsers to detect different formats by separating them with a comma.  | string ||
+| multiline.key_content| Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. | string ||

manifests/setup/fluentbit-operator-crd.yaml

@@ -361,6 +361,24 @@ spec:
                             type: object
                           type: array
                       type: object
+                    multiline:
+                      description: Multiline defines a Multiline configuration.
+                      properties:
+                        keyContent:
+                          description: Key name that holds the content to process.
+                            Note that a Multiline Parser definition can already specify
+                            the key_content to use, but this option allows to overwrite
+                            that value for the purpose of the filter.
+                          type: string
+                        parser:
+                          description: Specify one or multiple Multiline Parsing definitions
+                            to apply to the content. You can specify multiple multiline
+                            parsers to detect different formats by separating them
+                            with a comma.
+                          type: string
+                      required:
+                      - parser
+                      type: object
                     nest:
                       description: Nest defines Nest Filter configuration.
                       properties:

manifests/setup/setup.yaml

@@ -366,6 +366,24 @@ spec:
                             type: object
                           type: array
                       type: object
+                    multiline:
+                      description: Multiline defines a Multiline configuration.
+                      properties:
+                        keyContent:
+                          description: Key name that holds the content to process.
+                            Note that a Multiline Parser definition can already specify
+                            the key_content to use, but this option allows to overwrite
+                            that value for the purpose of the filter.
+                          type: string
+                        parser:
+                          description: Specify one or multiple Multiline Parsing definitions
+                            to apply to the content. You can specify multiple multiline
+                            parsers to detect different formats by separating them
+                            with a comma.
+                          type: string
+                      required:
+                      - parser
+                      type: object
                     nest:
                       description: Nest defines Nest Filter configuration.
                       properties: