Merge pull request #528 from lostsnow/feature/fluent-multi-line-parser

add fluent multi line parser
HXSecurity · May 30, 2023 · 13729c7 · 13729c7
2 parents 3d5a0e2 + fd2e740
commit 13729c7
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/LogCollector.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/LogCollector.java
@@ -22,6 +22,15 @@ public static void extractFluent() {
         }
         try {
             if (!isMacOs() && !isWindows()) {
+                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
+                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
+                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
+                FileUtils.confReplace(FLUENT_FILE_CONF);
+
+                String multiParserFile = IastProperties.getInstance().getTmpDir() + "parsers_multiline.conf";
+                FileUtils.getResourceToFile("bin/parsers_multiline.conf", multiParserFile);
+                FileUtils.confReplace(multiParserFile);
+
                 FLUENT_FILE = IastProperties.getInstance().getTmpDir() + "fluent";
                 File f = new File(FLUENT_FILE);
                 if (f.exists()) {
@@ -34,10 +43,6 @@ public static void extractFluent() {
                     FileUtils.getResourceToFile("bin/fluent", FLUENT_FILE);
                 }
 
-                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
-                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
-                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
-                FileUtils.confReplace(FLUENT_FILE_CONF);
                 if (!(new File(FLUENT_FILE)).setExecutable(true)) {
                     DongTaiLog.warn(ErrorCode.FLUENT_SET_EXECUTABLE_FAILED, FLUENT_FILE);
                 }

diff --git a/dongtai-agent/src/main/resources/bin/fluent.conf b/dongtai-agent/src/main/resources/bin/fluent.conf
@@ -3,6 +3,7 @@
   Daemon       OFF
   Log_Level    error
   HTTP_Server  Off
+  parsers_file parsers_multiline.conf
 [INPUT]
   Name         tail
   Path         ${LOG_PATH}
@@ -13,6 +14,7 @@
   Buffer_Max_Size   16MB
   Skip_Long_Lines   On
   Read_from_Head    true
+  multiline.parser multiline-regex-test
 [FILTER]
   Name         record_modifier
   Match        *

diff --git a/dongtai-agent/src/main/resources/bin/parsers_multiline.conf b/dongtai-agent/src/main/resources/bin/parsers_multiline.conf
@@ -0,0 +1,6 @@
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    rule      "start_state"   "/(\d+\d+\-\d+\-\d+ \d+\:\d+\:\d+)(.*)/"  "cont"
+    rule      "cont"          "/(^\s+at.*|^Caused.*|^\s+\.\.\..*)/"     "cont"