feature: 日志采集器支持同时采集容器多种类型日志 TencentBlueKing#372

zmberg · Feb 21, 2020 · f845ef1 · f845ef1
1 parent 81c1b99
commit f845ef1
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 40 deletions.
diff --git a/bcs-services/bcs-logbeat-sidecar/sidecar/controller.go b/bcs-services/bcs-logbeat-sidecar/sidecar/controller.go
@@ -30,18 +30,18 @@ import (
 
 const (
 	//report data id
-	EnvLogInfoDataid       = "io_tencent_bcs_app_dataid"
+	EnvLogInfoDataid       = "io_tencent_bcs_app_dataid_v2"
 	//if true, then stdout; else custom logs file
-	EnvLogInfoStdout       = "io_tencent_bcs_app_stdout"
+	EnvLogInfoStdout       = "io_tencent_bcs_app_stdout_v2"
 	//if stdout=false, log file path
-	EnvLogInfoLogPath      = "io_tencent_bcs_app_logpath"
+	EnvLogInfoLogPath      = "io_tencent_bcs_app_logpath_v2"
 	//clusterid
-	EnvLogInfoLogCluster   = "io_tencent_bcs_app_cluster"
+	EnvLogInfoLogCluster   = "io_tencent_bcs_app_cluster_v2"
 	//namespace
-	EnvLogInfoLogNamepsace = "io_tencent_bcs_app_namespcae"
+	EnvLogInfoLogNamepsace = "io_tencent_bcs_app_namespcae_v2"
 	//custom labels, log tags
 	//example: kv1:val1,kv2:val2,kv3:val3...
-	EnvLogInfoLogLabel     = "io_tencent_bcs_app_label"
+	EnvLogInfoLogLabel     = "io_tencent_bcs_app_label_v2"
 	//application or deployment't name
 	EnvLogInfoLogServerName = "io_tencent_bcs_controller_name"
 	//enum: Application、Deployment...
@@ -232,8 +232,8 @@ func produceLogConfParameter(container *docker.Container) (*LogConfParameter, bo
 					continue
 				}
 
-				label := fmt.Sprintf(`,
-            "%s": "%s"`, kvs[0], kvs[1])
+				label := fmt.Sprintf(`
+        %s: %s`, kvs[0], kvs[1])
 				para.CustemLabel += label
 			}
 		}
@@ -246,19 +246,30 @@ func produceLogConfParameter(container *docker.Container) (*LogConfParameter, bo
 		return nil, false
 	}
 
-	//if nonstandard log
-	if !para.stdout {
-		if para.nonstandardLog == "" {
-			blog.Warnf("container %s don't contain %s env", EnvLogInfoLogPath)
-			return nil, false
-		} else {
-			para.LogFile = para.nonstandardLog
+	files := make([]string,0)
+	//if stdout container log
+	if para.stdout {
+		files = append(files, container.LogPath)
+	}
+	//if nonstandard Log
+	if para.nonstandardLog!="" {
+		array := strings.Split(para.nonstandardLog, ",")
+		for _,f :=range array {
+			files = append(files, fmt.Sprintf("/proc/%d/root%s",container.State.Pid,f))
 		}
-		//else standard log
-	} else {
-		para.LogFile = container.LogPath
 	}
-
+	//if len(files)==0, then invalid
+	if len(files)==0 {
+		blog.Warnf("container %s env(%s, %s) is invalid",
+			container.ID, EnvLogInfoStdout, EnvLogInfoLogPath)
+		return nil, false
+	}
+	para.LogFile += files[0]
+	for _,f :=range files[1:] {
+		filelog := fmt.Sprintf(`
+        %s`, f)
+		para.LogFile += filelog
+	}
 	para.ContainerId = container.ID
 	return para, true
 }
@@ -274,7 +285,7 @@ func (s *SidecarController) produceContainerLogConf(c *docker.Container) {
 
 	logConf := &ContainerLogConf{
 		containerId: c.ID,
-		confPath:    fmt.Sprintf("%s/%s-%s.conf", s.conf.LogbeatDir, s.prefixFile, []byte(c.ID)[:12]),
+		confPath:    fmt.Sprintf("%s/%s-%s.yaml", s.conf.LogbeatDir, s.prefixFile, []byte(c.ID)[:12]),
 	}
 	para, ok := produceLogConfParameter(c)
 	if !ok {

diff --git a/bcs-services/bcs-logbeat-sidecar/template/unifytlogc-template.conf b/bcs-services/bcs-logbeat-sidecar/template/unifytlogc-template.conf
@@ -1,20 +1,12 @@
-{
-    "tlogcfg": [
-      {
-        "file": "{{.LogFile}}",
-        "dataid": {{.DataId}},
-        "private": [
-          {
-            "container_id": "{{.ContainerId}}",
-            "io.tencent.bcs.cluster": "{{.ClusterId}}",
-            "io.tencent.bcs.namespace": "{{.Namespace}}",
-            "io.tencent.bcs.server_name": "{{.ServerName}}",
-            "io.tencent.bcs.type": "{{.ServerType}}"{{.CustemLabel}}
-          }
-        ],
-        "field_sep": "|",
-        "fileds": [],
-        "beJson": 1
-      }
-    ]
-}
+local:
+  -
+    dataid: {{.DataId}}
+    paths:
+      - {{.LogFile}}
+    to_json: true
+    ext_meta:
+        container_id: {{.ContainerId}}
+        io.tencent.bcs.cluster: {{.ClusterId}}
+        io.tencent.bcs.namespace: {{.Namespace}}
+        io.tencent.bcs.server_name: {{.ServerName}}
+        io.tencent.bcs.type: {{.ServerType}}{{.CustemLabel}}