Merge pull request #534 from Scitizen/refactor/kubernetes-configs-split

Refactor/kubernetes configs split
fluent · Jul 7, 2021 · 0fab7ee · 0fab7ee
2 parents 1365189 + 83878ad
commit 0fab7ee
Show file tree

Hide file tree

Showing 17 changed files with 348 additions and 311 deletions.
diff --git a/Makefile b/Makefile
diff --git a/templates/Dockerfile.erb b/templates/Dockerfile.erb
@@ -55,6 +55,7 @@ COPY ./conf/systemd.conf /fluentd/etc/
 COPY ./conf/kubernetes.conf /fluentd/etc/
 COPY ./conf/prometheus.conf /fluentd/etc/
 COPY ./conf/tail_container_parse.conf /fluentd/etc/
+COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/
 RUN touch /fluentd/etc/disable.conf
 
 # Copy plugins

diff --git a/templates/conf/kubernetes.conf.erb b/templates/conf/kubernetes.conf.erb
@@ -20,172 +20,21 @@
   @include tail_container_parse.conf
 </source>
 
-<source>
-  @type tail
-  @id in_tail_minion
-  path /var/log/salt/minion
-  pos_file /var/log/fluentd-salt.pos
-  tag salt
-  <parse>
-    @type regexp
-    expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
-    time_format %Y-%m-%d %H:%M:%S
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_startupscript
-  path /var/log/startupscript.log
-  pos_file /var/log/fluentd-startupscript.log.pos
-  tag startupscript
-  <parse>
-    @type syslog
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_docker
-  path /var/log/docker.log
-  pos_file /var/log/fluentd-docker.log.pos
-  tag docker
-  <parse>
-    @type regexp
-    expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_etcd
-  path /var/log/etcd.log
-  pos_file /var/log/fluentd-etcd.log.pos
-  tag etcd
-  <parse>
-    @type none
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_kubelet
-  multiline_flush_interval 5s
-  path /var/log/kubelet.log
-  pos_file /var/log/fluentd-kubelet.log.pos
-  tag kubelet
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_kube_proxy
-  multiline_flush_interval 5s
-  path /var/log/kube-proxy.log
-  pos_file /var/log/fluentd-kube-proxy.log.pos
-  tag kube-proxy
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_kube_apiserver
-  multiline_flush_interval 5s
-  path /var/log/kube-apiserver.log
-  pos_file /var/log/fluentd-kube-apiserver.log.pos
-  tag kube-apiserver
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
+@include kubernetes/cluster-autoscaler.conf
+@include kubernetes/containers.conf
+@include kubernetes/docker.conf
+@include kubernetes/etcd.conf
+@include kubernetes/glbc.conf
+@include kubernetes/kube-apiserver-audit.conf
+@include kubernetes/kube-apiserver.conf
+@include kubernetes/kube-controller-manager.conf
+@include kubernetes/kube-proxy.conf
+@include kubernetes/kube-scheduler.conf
+@include kubernetes/kubelet.conf
+@include kubernetes/rescheduler.conf
+@include kubernetes/salt.conf
+@include kubernetes/startupscript.conf
 
-<source>
-  @type tail
-  @id in_tail_kube_controller_manager
-  multiline_flush_interval 5s
-  path /var/log/kube-controller-manager.log
-  pos_file /var/log/fluentd-kube-controller-manager.log.pos
-  tag kube-controller-manager
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_kube_scheduler
-  multiline_flush_interval 5s
-  path /var/log/kube-scheduler.log
-  pos_file /var/log/fluentd-kube-scheduler.log.pos
-  tag kube-scheduler
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_rescheduler
-  multiline_flush_interval 5s
-  path /var/log/rescheduler.log
-  pos_file /var/log/fluentd-rescheduler.log.pos
-  tag rescheduler
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_glbc
-  multiline_flush_interval 5s
-  path /var/log/glbc.log
-  pos_file /var/log/fluentd-glbc.log.pos
-  tag glbc
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-<source>
-  @type tail
-  @id in_tail_cluster_autoscaler
-  multiline_flush_interval 5s
-  path /var/log/cluster-autoscaler.log
-  pos_file /var/log/fluentd-cluster-autoscaler.log.pos
-  tag cluster-autoscaler
-  <parse>
-    @type kubernetes
-  </parse>
-</source>
-
-# Example:
-# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
-# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
-<source>
-  @type tail
-  @id in_tail_kube_apiserver_audit
-  multiline_flush_interval 5s
-  path /var/log/kubernetes/kube-apiserver-audit.log
-  pos_file /var/log/kube-apiserver-audit.log.pos
-  tag kube-apiserver-audit
-  <parse>
-    @type multiline
-    format_firstline /^\S+\s+AUDIT:/
-    # Fields must be explicitly captured by name to be parsed into the record.
-    # Fields may not always be present, and order may change, so this just looks
-    # for a list of key="\"quoted\" value" pairs separated by spaces.
-    # Unknown fields are ignored.
-    # Note: We can't separate query/response lines as format1/format2 because
-    #       they don't always come one after the other for a given query.
-    format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
-    time_format %Y-%m-%dT%T.%L%Z
-  </parse>
-</source>
 
 <filter kubernetes.**>
   @type kubernetes_metadata

diff --git a/templates/conf/kubernetes/cluster-autoscaler.conf.erb b/templates/conf/kubernetes/cluster-autoscaler.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_cluster_autoscaler
+  multiline_flush_interval 5s
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/fluentd-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/containers.conf.erb b/templates/conf/kubernetes/containers.conf.erb
@@ -0,0 +1,14 @@
+
+<source>
+@type tail
+@id in_tail_container_logs
+path /var/log/containers/*.log
+pos_file /var/log/fluentd-containers.log.pos
+tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
+exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}"
+read_from_head true
+<parse>
+  @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+</parse>
+</source>
diff --git a/templates/conf/kubernetes/docker.conf.erb b/templates/conf/kubernetes/docker.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_docker
+  path /var/log/docker.log
+  pos_file /var/log/fluentd-docker.log.pos
+  tag docker
+  <parse>
+    @type regexp
+    expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/etcd.conf.erb b/templates/conf/kubernetes/etcd.conf.erb
@@ -0,0 +1,10 @@
+<source>
+  @type tail
+  @id in_tail_etcd
+  path /var/log/etcd.log
+  pos_file /var/log/fluentd-etcd.log.pos
+  tag etcd
+  <parse>
+    @type none
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/glbc.conf.erb b/templates/conf/kubernetes/glbc.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_glbc
+  multiline_flush_interval 5s
+  path /var/log/glbc.log
+  pos_file /var/log/fluentd-glbc.log.pos
+  tag glbc
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kube-apiserver-audit.conf.erb b/templates/conf/kubernetes/kube-apiserver-audit.conf.erb
@@ -0,0 +1,23 @@
+# Example:
+# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
+# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
+<source>
+  @type tail
+  @id in_tail_kube_apiserver_audit
+  multiline_flush_interval 5s
+  path /var/log/kubernetes/kube-apiserver-audit.log
+  pos_file /var/log/kube-apiserver-audit.log.pos
+  tag kube-apiserver-audit
+  <parse>
+    @type multiline
+    format_firstline /^\S+\s+AUDIT:/
+    # Fields must be explicitly captured by name to be parsed into the record.
+    # Fields may not always be present, and order may change, so this just looks
+    # for a list of key="\"quoted\" value" pairs separated by spaces.
+    # Unknown fields are ignored.
+    # Note: We can't separate query/response lines as format1/format2 because
+    #       they don't always come one after the other for a given query.
+    format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
+    time_format %Y-%m-%dT%T.%L%Z
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kube-apiserver.conf.erb b/templates/conf/kubernetes/kube-apiserver.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_kube_apiserver
+  multiline_flush_interval 5s
+  path /var/log/kube-apiserver.log
+  pos_file /var/log/fluentd-kube-apiserver.log.pos
+  tag kube-apiserver
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kube-controller-manager.conf.erb b/templates/conf/kubernetes/kube-controller-manager.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_kube_controller_manager
+  multiline_flush_interval 5s
+  path /var/log/kube-controller-manager.log
+  pos_file /var/log/fluentd-kube-controller-manager.log.pos
+  tag kube-controller-manager
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kube-proxy.conf.erb b/templates/conf/kubernetes/kube-proxy.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_kube_proxy
+  multiline_flush_interval 5s
+  path /var/log/kube-proxy.log
+  pos_file /var/log/fluentd-kube-proxy.log.pos
+  tag kube-proxy
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kube-scheduler.conf.erb b/templates/conf/kubernetes/kube-scheduler.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_kube_scheduler
+  multiline_flush_interval 5s
+  path /var/log/kube-scheduler.log
+  pos_file /var/log/fluentd-kube-scheduler.log.pos
+  tag kube-scheduler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/kubelet.conf.erb b/templates/conf/kubernetes/kubelet.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_kubelet
+  multiline_flush_interval 5s
+  path /var/log/kubelet.log
+  pos_file /var/log/fluentd-kubelet.log.pos
+  tag kubelet
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/rescheduler.conf.erb b/templates/conf/kubernetes/rescheduler.conf.erb
@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_rescheduler
+  multiline_flush_interval 5s
+  path /var/log/rescheduler.log
+  pos_file /var/log/fluentd-rescheduler.log.pos
+  tag rescheduler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/salt.conf.erb b/templates/conf/kubernetes/salt.conf.erb
@@ -0,0 +1,13 @@
+
+<source>
+  @type tail
+  @id in_tail_minion
+  path /var/log/salt/minion
+  pos_file /var/log/fluentd-salt.pos
+  tag salt
+  <parse>
+    @type regexp
+    expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
+    time_format %Y-%m-%d %H:%M:%S
+  </parse>
+</source>
diff --git a/templates/conf/kubernetes/startupscript.conf.erb b/templates/conf/kubernetes/startupscript.conf.erb
@@ -0,0 +1,10 @@
+<source>
+  @type tail
+  @id in_tail_startupscript
+  path /var/log/startupscript.log
+  pos_file /var/log/fluentd-startupscript.log.pos
+  tag startupscript
+  <parse>
+    @type syslog
+  </parse>
+</source>