Fluentd to Elasticsearch image

contrib/logging/fluentd-es-image/Dockerfile

@@ -0,0 +1,42 @@
+# This Dockerfile will build an image that is configured
+# to run Fluentd with an Elasticsearch plug-in and the
+# provided configuration file.
+# TODO(satnam6502): Use a lighter base image, e.g. some form of busybox.
+# The image acts as an executable for the binary /usr/sbin/td-agent.
+# Note that fluentd is run with root permssion to allow access to
+# log files with root only access under /var/lib/docker/containers/*
+# Please see http://docs.fluentd.org/articles/install-by-deb for more
+# information about installing fluentd using deb package.
+
+FROM ubuntu:14.04
+MAINTAINER Satnam Singh "satnam@google.com"
+
+# Ensure there are enough file descriptors for running Fluentd.
+RUN ulimit -n 65536
+
+# Install prerequisites.
+RUN apt-get update && \
+    apt-get install -y curl && \
+    apt-get install -y -q libcurl4-openssl-dev make && \
+    apt-get clean
+
+# Install Fluentd.
+RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh
+
+# Change the default user and group to root.
+# Needed to allow access to /var/log/docker/... files.
+RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent
+
+# Install the Elasticsearch Fluentd plug-in.
+RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch
+
+# Copy the Fluentd configuration file.
+COPY td-agent.conf /etc/td-agent/td-agent.conf
+
+# Copy a script that determines the name of the host machine
+# and then patch the Fluentd configuration files and then
+# run Fluentd in the foreground.
+ADD run.sh /run.sh
+
+# Always run the this setup script.
+ENTRYPOINT ["/run.sh"]

contrib/logging/fluentd-es-image/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Build the fluentd-elasticsearch image and push
+# to google/fluentd-elasticsearch.
+
+sudo docker build -t kubernetes/fluentd-elasticsearch .
+sudo docker push kubernetes/fluentd-elasticsearch

contrib/logging/fluentd-es-image/run.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING! HORRIBLE HACK! We expect /outerhost to be mapped to
+# the enclosing /etc/host file so we can determine the name of
+# the host machine (super fragile). This is a temporary hack until
+# service IPs are done.
+OUTER_HOST=`tail -n 1 /outerhost | awk '{print $3}'`
+
+# Copy the Fluentd config file and patch it to refer to the
+# name of the host machine for ES_HOST. HACK!
+cp td-agent.conf /etc/td-agent
+sed -i -e "s/ES_HOST/${OUTER_HOST}/" /etc/td-agent/td-agent.conf
+/usr/sbin/td-agent

contrib/logging/fluentd-es-image/td-agent.conf

@@ -0,0 +1,53 @@
+# This configuration file for Fluentd / td-agent is used
+# to watch changes to Docker log files that live in the
+# directory /var/lib/docker/containers/ which are then submitted to
+# Elasticsearch (running on the machine ES_HOST:9200) which
+# assumes the installation of the fluentd-elasticsearch plug-in.
+# See https://github.com/uken/fluent-plugin-elasticsearch for
+# more information about the plug-in. This file needs to be
+# patched to replace ES_HOST with the name of the actual
+# machine running Elasticsearch.
+# Maintainer: Satnam Singh (satnam@google.com)
+#
+# Exampe
+# ======
+# A line in the Docker log file might like like this JSON:
+#
+# {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
+#  "stream":"stderr",
+#   "time":"2014-09-25T21:15:03.499185026Z"}
+#
+# The time_format specification below makes sure we properly
+# parse the time format produced by Docker. This will be
+# submitted to Elasticsearch and should appear like:
+# $ curl 'http://elasticsearch:9200/_search?pretty'
+# ...
+# {
+#      "_index" : "logstash-2014.09.25",
+#      "_type" : "fluentd",
+#      "_id" : "VBrbor2QTuGpsQyTCdfzqA",
+#      "_score" : 1.0,
+#      "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n",
+#                 "stream":"stderr","tag":"docker.container.all",
+#                 "@timestamp":"2014-09-25T22:45:50+00:00"}
+#    },
+# ...
+
+<source>
+  type tail
+  format json
+  time_key time
+  path /var/lib/docker/containers/*/*-json.log
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.container.all
+</source>
+
+<match docker.container.*>
+   type elasticsearch
+   log_level info
+   include_tag_key true
+   host ES_HOST
+   port 9200
+   logstash_format true
+   flush_interval 5s
+</match>