From 6215089796b62ef08ad4650b47c72220672dd9ee Mon Sep 17 00:00:00 2001
From: Mik Vyatskov <vmik@google.com>
Date: Wed, 30 Nov 2016 11:41:54 +0100
Subject: [PATCH] Unify fluentd-gcp configurations

---
 .../fluentd-gcp/fluentd-gcp-image/Dockerfile  |  12 +-
 .../fluentd-gcp/fluentd-gcp-image/Makefile    |  17 +-
 ...oogle-fluentd-journal.conf => fluent.conf} |   8 -
 .../fluentd-gcp-image/google-fluentd.conf     | 272 ------------------
 .../fluentd-gcp/fluentd-gcp-image/run.sh      |  29 ++
 5 files changed, 42 insertions(+), 296 deletions(-)
 rename cluster/addons/fluentd-gcp/fluentd-gcp-image/{google-fluentd-journal.conf => fluent.conf} (97%)
 delete mode 100644 cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
 create mode 100755 cluster/addons/fluentd-gcp/fluentd-gcp-image/run.sh

diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile
index 203cc45288cd6..164cc1de5b26e 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile
@@ -50,11 +50,11 @@ RUN apt-get -qq update && \
         /opt/td-agent/embedded/share/postgresql \
         /etc/td-agent/td-agent.conf
 
-# Copy the Fluentd configuration files for logging Docker container logs.
-# Either configuration file can be used by specifying `-c <file>` as a command
-# line argument.
-COPY google-fluentd.conf /etc/td-agent/td-agent.conf
-COPY google-fluentd-journal.conf /etc/td-agent/td-agent-journal.conf
+# Copy the Fluentd configuration file for logging Docker container logs.
+COPY fluent.conf /etc/td-agent/td-agent.conf
+
+# Copy the entrypoint for the container
+COPY run.sh /run.sh
 
 # Start Fluentd to pick up our config that watches Docker container logs.
-CMD /usr/sbin/td-agent $FLUENTD_ARGS
+CMD /run.sh $FLUENTD_ARGS
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
index 703bb2de2a72e..400d8b2210632 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
@@ -13,23 +13,20 @@
 # limitations under the License.
 
 # The build rule builds a Docker image that logs all Docker contains logs to
-# Google Compute Platform using the Cloud Logging API. The push rule pushes
-# the image to DockerHub.
+# Google Compute Platform using the Cloud Logging API.
 
 # Procedure for change:
 # 1. Bump the tag number.
-# 2. Create an image for the kubernetes user on Dockerhub and push there: make kbuild kpush
-# 3. Temporarily adjust any config files etc. that reference gcr.io/google_containers/fluentd-gcp:$(TAG)
-#    to use kubernetes/fluentd-gcp:$(TAG) instead.
-# 4. Test etc.
-# 5. Issue PR.
-# 6. When PR is approved make the gcr.io version of the image: make build push
-# 7. Revert the referendes to kubernetes/fluentd-gcp:$(TAG) to gcr.io/google_containers/fluentd-gcp:$(TAG)
+# 2. Push to the private repo and test using newer version
+# 3. Issue PR.
+# 4. Assuming permissions to do so, when PR is approved
+#    make the gcr.io version of the image: make build push
+# 5. Issue PR with config files changes
 
 .PHONY:	build push
 
 PREFIX=gcr.io/google_containers
-TAG = 1.28
+TAG = 1.29
 
 build:
 	docker build -t $(PREFIX)/fluentd-gcp:$(TAG) .
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/fluent.conf
similarity index 97%
rename from cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf
rename to cluster/addons/fluentd-gcp/fluentd-gcp-image/fluent.conf
index d3b7cdfc3c75c..3c6b01e89e880 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/fluent.conf
@@ -5,9 +5,6 @@
 # pod name and container name. These logs are then submitted to
 # Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
 #
-# This configuration is almost identical to google-fluentd.conf, with the one
-# difference being that this collects systemd journal logs.
-#
 # Example
 # =======
 # A line in the Docker log file might like like this JSON:
@@ -42,11 +39,6 @@
 # the name of the Kubernetes container regardless of how many times the
 # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
 
-# Do not directly collect fluentd's own logs to avoid infinite loops.
-<match fluent.**>
-  type null
-</match>
-
 # Example:
 # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
 <source>
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
deleted file mode 100644
index f5979201c1599..0000000000000
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
+++ /dev/null
@@ -1,272 +0,0 @@
-# This configuration file for Fluentd / td-agent is used
-# to watch changes to Docker log files that live in the
-# directory /var/lib/docker/containers/ and are symbolically
-# linked to from the /var/log directory using names that capture the
-# pod name and container name. These logs are then submitted to
-# Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
-#
-# This configuration is almost identical to google-fluentd-journal.conf, with
-# the one difference being that this doesn't try to collect systemd journal
-# logs.
-#
-# Example
-# =======
-# A line in the Docker log file might like like this JSON:
-#
-# {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
-#  "stream":"stderr",
-#   "time":"2014-09-25T21:15:03.499185026Z"}
-#
-# The record reformer is used to write the tag to focus on the pod name
-# and the Kubernetes container name. For example a Docker container's logs
-# might be in the directory:
-#  /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
-# and in the file:
-#  997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
-# where 997599971ee6... is the Docker ID of the running container.
-# The Kubernetes kubelet makes a symbolic link to this file on the host machine
-# in the /var/log/containers directory which includes the pod name and the Kubernetes
-# container name:
-#    synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 
-#    -> 
-#    /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
-# The /var/log directory on the host is mapped to the /var/log directory in the container
-# running this instance of Fluentd and we end up collecting the file:
-#   /var/log/containers/synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
-# This results in the tag:
-#  var.log.containers.synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
-# The record reformer is used is discard the var.log.containers prefix and
-# the Docker container ID suffix and "kubernetes." is pre-pended giving the
-# final tag which is ingested into Elasticsearch:
-#   kubernetes.synthetic-logger-0.25lps-pod_default-synth-lgr
-# This makes it easier for users to search for logs by pod name or by
-# the name of the Kubernetes container regardless of how many times the
-# Kubernetes pod has been restarted (resulting in a several Docker container IDs).
-
-# Do not directly collect fluentd's own logs to avoid infinite loops.
-<match fluent.**>
-  type null
-</match>
-
-# Example:
-# {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
-<source>
-  type tail
-  format json
-  time_key time
-  path /var/log/containers/*.log
-  pos_file /var/log/gcp-containers.log.pos
-  time_format %Y-%m-%dT%H:%M:%S.%NZ
-  tag reform.*
-  read_from_head true
-</source>
-
-<match reform.**>
-  type record_reformer
-  enable_ruby true
-  tag kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
-</match>
-
-# Example:
-# 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
-<source>
-  type tail
-  format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
-  time_format %Y-%m-%d %H:%M:%S
-  path /var/log/salt/minion
-  pos_file /var/log/gcp-salt.pos
-  tag salt
-</source>
-
-# Example:
-# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
-<source>
-  type tail
-  format syslog
-  path /var/log/startupscript.log
-  pos_file /var/log/gcp-startupscript.log.pos
-  tag startupscript
-</source>
-
-# Examples:
-# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
-# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
-<source>
-  type tail
-  format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
-  path /var/log/docker.log
-  pos_file /var/log/gcp-docker.log.pos
-  tag docker
-</source>
-
-# Example:
-# 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
-<source>
-  type tail
-  # Not parsing this, because it doesn't have anything particularly useful to
-  # parse out of it (like severities).
-  format none
-  path /var/log/etcd.log
-  pos_file /var/log/gcp-etcd.log.pos
-  tag etcd
-</source>
-
-# Multi-line parsing is required for all the kube logs because very large log
-# statements, such as those that include entire object bodies, get split into
-# multiple lines by glog.
-
-# Example:
-# I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/kubelet.log
-  pos_file /var/log/gcp-kubelet.log.pos
-  tag kubelet
-</source>
-
-# Example:
-# I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/kube-proxy.log
-  pos_file /var/log/gcp-kube-proxy.log.pos
-  tag kube-proxy
-</source>
-
-# Example:
-# I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/kube-apiserver.log
-  pos_file /var/log/gcp-kube-apiserver.log.pos
-  tag kube-apiserver
-</source>
-
-# Example:
-# I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/kube-controller-manager.log
-  pos_file /var/log/gcp-kube-controller-manager.log.pos
-  tag kube-controller-manager
-</source>
-
-# Example:
-# W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/kube-scheduler.log
-  pos_file /var/log/gcp-kube-scheduler.log.pos
-  tag kube-scheduler
-</source>
-
-# Example:
-# I1104 10:36:20.242766       5 rescheduler.go:73] Running Rescheduler
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/rescheduler.log
-  pos_file /var/log/gcp-rescheduler.log.pos
-  tag rescheduler
-</source>
-
-# Example:
-# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/glbc.log
-  pos_file /var/log/gcp-glbc.log.pos
-  tag glbc
-</source>
-
-# Example:
-# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
-<source>
-  type tail
-  format multiline
-  multiline_flush_interval 5s
-  format_firstline /^\w\d{4}/
-  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
-  time_format %m%d %H:%M:%S.%N
-  path /var/log/cluster-autoscaler.log
-  pos_file /var/log/gcp-cluster-autoscaler.log.pos
-  tag cluster-autoscaler
-</source>
-
-# We use 2 output stanzas - one to handle the container logs and one to handle
-# the node daemon logs, the latter of which explicitly sends its logs to the
-# compute.googleapis.com service rather than container.googleapis.com to keep
-# them separate since most users don't care about the node logs.
-<match kubernetes.**>
-  type google_cloud
-  # Set the buffer type to file to improve the reliability and reduce the memory consumption
-  buffer_type file
-  buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
-  # Set queue_full action to block because we want to pause gracefully
-  # in case of the off-the-limits load instead of throwing an exception
-  buffer_queue_full_action block
-  # Set the chunk limit conservatively to avoid exceeding the GCL limit
-  # of 10MiB per write request.
-  buffer_chunk_limit 2M
-  # Cap the combined memory usage of this buffer and the one below to
-  # 2MiB/chunk * (6 + 2) chunks = 16 MiB
-  buffer_queue_limit 6
-  # Never wait more than 5 seconds before flushing logs in the non-error case.
-  flush_interval 5s
-  # Never wait longer than 30 seconds between retries.
-  max_retry_wait 30
-  # Disable the limit on the number of retries (retry forever).
-  disable_retry_limit
-  # Use multiple threads for processing.
-  num_threads 2
-</match>
-
-# Keep a smaller buffer here since these logs are less important than the user's
-# container logs.
-<match **>
-  type google_cloud
-  detect_subservice false
-  buffer_type file
-  buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
-  buffer_queue_full_action block
-  buffer_chunk_limit 2M
-  buffer_queue_limit 2
-  flush_interval 5s
-  max_retry_wait 30
-  disable_retry_limit
-  num_threads 2
-</match>
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/run.sh b/cluster/addons/fluentd-gcp/fluentd-gcp-image/run.sh
new file mode 100755
index 0000000000000..09f09508b720c
--- /dev/null
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/run.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For systems without journald
+mkdir -p /var/log/journal
+
+if [ -e /host/lib/libsystemd* ]
+then
+  rm /lib/x86_64-linux-gnu/libsystemd*
+  cp /host/lib/libsystemd* /lib/x86_64-linux-gnu/
+fi
+
+LD_PRELOAD=/opt/td-agent/embedded/lib/libjemalloc.so
+RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9
+
+/usr/sbin/td-agent "$@"