Adding splunk HEC support

Makefile

@@ -25,6 +25,7 @@ ALL_IMAGES := \
 	v0.12/alpine-logzio:v0.12.43-logzio,v0.12-logzio,stable-logzio,logzio \
 	v0.12/alpine-kafka:v0.12.43-kafka,v0.12-kafka,stable-kafka,kafka \
 	v0.12/alpine-kinesis:v0.12.43-kinesis,v0.12-kinesis,stable-kinesis,kinesis \
+	v0.12/alpine-splunkhec:v0.12.43-splunkhec,v0.12-splunkhec,stable-splunkhec,splunkhec \
 	v0.12/debian-elasticsearch:v0.12.43-debian-elasticsearch,v0.12-debian-elasticsearch,debian-elasticsearch \
 	v0.12/debian-loggly:v0.12.43-debian-loggly,v0.12-debian-loggly,debian-loggly \
 	v0.12/debian-logentries:v0.12.43-debian-logentries,v0.12-debian-logentries,debian-logentries \
@@ -37,6 +38,7 @@ ALL_IMAGES := \
 	v0.12/debian-graylog:v0.12.43-debian-graylog,v0.12-debian-graylog,debian-stable-graylog,debian-graylog \
 	v0.12/debian-logzio:v0.12.43-debian-logzio,v0.12-debian-logzio,debian-logzio \
 	v0.12/debian-kafka:v0.12.43-debian-kafka,v0.12-debian-kafka,debian-kafka \
+	v0.12/debian-splunkhec:v0.12.43-debian-splunkhec,v0.12-debian-splunkhec,debian-splunkhec \
 	v0.12/debian-kinesis:v0.12.43-debian-kinesis,v0.12-debian-kinesis,debian-kinesis \
 	v1.2/debian-elasticsearch:v1.2.5-debian-elasticsearch,v1.2-debian-elasticsearch \
 	v1.2/debian-loggly:v1.2.5-debian-loggly,v1.2-debian-loggly \

README.md

@@ -22,6 +22,7 @@
 - `v0.12-debian-graylog` [docker-image/v0.12/debian-graylog/Dockerfile](docker-image/v0.12/debian-graylog/Dockerfile)
 - `v0.12-debian-logzio` [docker-image/v0.12/debian-logzio/Dockerfile](docker-image/v0.12/debian-logzio/Dockerfile)
 - `v0.12-debian-kafka` [docker-image/v0.12/debian-kafka/Dockerfile](docker-image/v0.12/debian-kafka/Dockerfile)
+- `v0.12-debian-splunkhec` [docker-image/v0.12/debian-splunkhec/Dockerfile](docker-image/v0.12/debian-splunkhec/Dockerfile)
 - `v0.12-debian-kinesis` [docker-image/v0.12/debian-kinesis/Dockerfile](docker-image/v0.12/debian-kinesis/Dockerfile)
 - `v1.2-debian-elasticsearch` [docker-image/v1.2/debian-elasticsearch/Dockerfile](docker-image/v1.2/debian-elasticsearch/Dockerfile)
 - `v1.2-debian-loggly` [docker-image/v1.2/debian-loggly/Dockerfile](docker-image/v1.2/debian-loggly/Dockerfile)
@@ -51,6 +52,7 @@
 - `v0.12-alpine-graylog` [docker-image/v0.12/alpine-graylog/Dockerfile](docker-image/v0.12/alpine-graylog/Dockerfile)
 - `v0.12-alpine-logzio` [docker-image/v0.12/alpine-logzio/Dockerfile](docker-image/v0.12/alpine-logzio/Dockerfile)
 - `v0.12-alpine-kafka` [docker-image/v0.12/alpine-kafka/Dockerfile](docker-image/v0.12/alpine-kafka/Dockerfile)
+- `v0.12-alpine-splunkhec` [docker-image/v0.12/alpine-splunkhec/Dockerfile](docker-image/v0.12/alpine-splunkhec/Dockerfile)
 - `v0.12-alpine-kinesis` [docker-image/v0.12/alpine-kinesis/Dockerfile](docker-image/v0.12/alpine-kinesis/Dockerfile)
 
 ## What is Fluentd?
@@ -92,6 +94,7 @@ Some images are contributed by users. If you have a problem/question for followi
 - gcs : @andor-pierdelacabeza
 - Amazon Kinesis : @shiftky
 - logz.io : @SaMnCo / @jamielennox
+- splunkhec: @FutureSharks
 
 ### References
 

docker-image/v0.12/alpine-splunkhec/.dockerignore

@@ -0,0 +1 @@
+**/*.gitkeep

docker-image/v0.12/alpine-splunkhec/Dockerfile

@@ -0,0 +1,48 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
+
+FROM fluent/fluentd:v0.12.43
+LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>"
+USER root
+WORKDIR /home/fluent
+ENV PATH /fluentd/vendor/bundle/ruby/2.4.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.4.0
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.4.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+RUN set -ex \
+    && apk upgrade --no-cache \
+    && apk add --no-cache ruby-bundler \
+    && apk add --no-cache --virtual .build-deps \
+        build-base \
+        ruby-dev \
+        libffi-dev \
+    && gem install bundler --version 1.16.1 \
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
+    && apk del .build-deps \
+    && gem sources --clear-all \
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+# Copy configuration files
+COPY ./conf/fluent.conf /fluentd/etc/
+COPY ./conf/kubernetes.conf /fluentd/etc/
+
+# Copy plugins
+COPY plugins /fluentd/plugins/
+COPY entrypoint.sh /fluentd/entrypoint.sh
+
+# Environment variables
+ENV FLUENTD_OPT=""
+ENV FLUENTD_CONF="fluent.conf"
+
+# jemalloc is memory optimization only available for td-agent
+# td-agent is provided and QA'ed by treasuredata as rpm/deb/.. package
+# -> td-agent (stable) vs fluentd (edge)
+#ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+
+# Run Fluentd
+CMD ["/fluentd/entrypoint.sh"]

docker-image/v0.12/alpine-splunkhec/Gemfile

@@ -0,0 +1,12 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "0.12.43"
+gem "oj", "3.5.1"
+gem "fluent-plugin-secure-forward"
+gem "fluent-plugin-record-reformer"
+gem "fluent-plugin-rewrite-tag-filter"
+gem "fluent-plugin-splunk-enterprise"
+gem "fluent-plugin-kubernetes_metadata_filter"

docker-image/v0.12/alpine-splunkhec/Gemfile.lock

@@ -0,0 +1,100 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    concurrent-ruby (1.0.5)
+    cool.io (1.5.3)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    fluent-plugin-kubernetes_metadata_filter (1.2.0)
+      fluentd (>= 0.12.0)
+      kubeclient (~> 1.1.4)
+      lru_redux
+    fluent-plugin-record-reformer (0.9.1)
+      fluentd
+    fluent-plugin-rewrite-tag-filter (1.6.0)
+      fluentd (>= 0.12.0, < 0.14.0)
+    fluent-plugin-secure-forward (0.4.5)
+      fluentd (>= 0.10.46)
+      proxifier
+      resolve-hostname
+    fluent-plugin-splunk-enterprise (0.9.1)
+      fluentd (>= 0.12.0)
+      httpclient
+      json
+    fluentd (0.12.43)
+      cool.io (>= 1.2.2, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.5.11, < 2)
+      sigdump (~> 0.2.2)
+      string-scrub (>= 0.0.3, <= 0.0.5)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http (0.9.8)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    httpclient (2.8.3)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    json (2.1.0)
+    kubeclient (1.1.4)
+      activesupport
+      http (= 0.9.8)
+      recursive-open-struct (= 1.0.0)
+      rest-client
+    lru_redux (1.1.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    minitest (5.11.3)
+    msgpack (1.2.4)
+    netrc (0.11.0)
+    oj (3.5.1)
+    proxifier (1.0.3)
+    public_suffix (3.0.3)
+    recursive-open-struct (1.0.0)
+    resolve-hostname (0.1.0)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    sigdump (0.2.4)
+    string-scrub (0.0.5)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.5)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-kubernetes_metadata_filter
+  fluent-plugin-record-reformer
+  fluent-plugin-rewrite-tag-filter
+  fluent-plugin-secure-forward
+  fluent-plugin-splunk-enterprise
+  fluentd (= 0.12.43)
+  oj (= 3.5.1)
+
+BUNDLED WITH
+   1.16.3

docker-image/v0.12/alpine-splunkhec/conf/fluent.conf

@@ -0,0 +1,33 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+@include kubernetes.conf
+
+<match **>
+  @type splunk_hec
+  host "#{ENV['FLUENT_SPLUNK_HEC_HOST'] || nil}"
+  port "#{ENV['FLUENT_SPLUNK_HEC_PORT'] || '8089'}"
+  token "#{ENV['FLUENT_SPLUNK_HEC_TOKEN'] || nil}"
+
+  # metadata parameter
+  default_source "#{ENV['FLUENT_SPLUNK_HEC_SOURCE'] || 'fluentd-splunkhec'}"
+  sourcetype "#{ENV['FLUENT_SPLUNK_HEC_SOURCE_TYPE'] || 'fluentd-splunkhec'}"
+
+  use_fluentd_time "#{ENV['FLUENT_SPLUNK_HEC_USE_FLUENTD_TIME'] || true}"
+
+  # ack parameter
+  use_ack "#{ENV['FLUENT_SPLUNK_HEC_USE_ACK'] || false}"
+  ack_interval "#{ENV['FLUENT_SPLUNK_HEC_ACK_INTERVAL'] || '1'}"
+  ack_retry_limit "#{ENV['FLUENT_SPLUNK_HEC_ACK_RETRY_LIMIT'] || '3'}"
+
+  # ssl parameter
+  use_ssl "#{ENV['FLUENT_SPLUNK_HEC_USE_SSL'] || true}"
+  ssl_verify "#{ENV['FLUENT_SPLUNK_HEC_SSL_VERIFY'] || true}"
+  ca_file "#{ENV['FLUENT_SPLUNK_HEC_CA_FILE'] || '/etc/ssl/certs/ca-bundle.crt'}"
+
+  # buffered output parameter
+  flush_interval "#{ENV['FLUENT_SPLUNK_HEC_FLUSH_INTERVAL'] || '10s'}"
+
+</match>
+