multiline support

Signed-off-by: chengdehao <dehaocheng@yunify.com>
fluent · Nov 23, 2021 · aeb0ab9 · aeb0ab9
1 parent d674530
commit aeb0ab9
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 0 deletions.
diff --git a/api/fluentbitoperator/v1alpha2/plugins/input/tail_types.go b/api/fluentbitoperator/v1alpha2/plugins/input/tail_types.go
@@ -87,6 +87,8 @@ type Tail struct {
 	DockerModeFlushSeconds *int64 `json:"dockerModeFlushSeconds,omitempty"`
 	// DisableInotifyWatcher will disable inotify and use the file stat watcher instead.
 	DisableInotifyWatcher *bool `json:"disableInotifyWatcher,omitempty"`
+	// This will help to reassembly multiline messages originally split by Docker or CRI
+	MultilineParser string `json:"multilineParser,omitempty"`
 }
 
 func (_ *Tail) Name() string {
@@ -167,5 +169,8 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if t.DisableInotifyWatcher != nil {
 		kvs.Insert("Inotify_Watcher", fmt.Sprint(!*t.DisableInotifyWatcher))
 	}
+	if t.MultilineParser != "" {
+		kvs.Insert("multiline.parser", t.MultilineParser)
+	}
 	return kvs, nil
 }
diff --git a/conf/parsers.conf b/conf/parsers.conf
@@ -95,6 +95,24 @@
     Time_Keep   On
     Time_Key time
 
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    #
+    # Regex rules for multiline parsing
+    # ---------------------------------
+    #
+    # configuration hints:
+    #
+    #  - first state always has the name: start_state
+    #  - every field in the rule must be inside double quotes
+    #
+    # rules |   state name  | regex pattern                  | next state
+    # ------|---------------|--------------------------------------------
+    rule      "start_state"   "/(Dec \d+ \d+\:\d+\:\d+)(.*)/"  "cont"
+    rule      "cont"          "/^\s+at.*/"                     "cont"
+
 [PARSER]
     # https://rubular.com/r/3fVxCrE5iFiZim
     Name    envoy

diff --git a/config/crd/bases/logging.kubesphere.io_inputs.yaml b/config/crd/bases/logging.kubesphere.io_inputs.yaml
@@ -212,6 +212,10 @@ spec:
                       messages
                     format: int64
                     type: integer
+                  multilineParser:
+                    description: This will help to reassembly multiline messages originally
+                      split by Docker or CRI
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

diff --git a/manifests/setup/fluentbit-operator-crd.yaml b/manifests/setup/fluentbit-operator-crd.yaml
@@ -3315,6 +3315,10 @@ spec:
                       messages
                     format: int64
                     type: integer
+                  multilineParser:
+                    description: This will help to reassembly multiline messages originally
+                      split by Docker or CRI
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml
@@ -3320,6 +3320,10 @@ spec:
                       messages
                     format: int64
                     type: integer
+                  multilineParser:
+                    description: This will help to reassembly multiline messages originally
+                      split by Docker or CRI
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.