Merge b4f6556 into ff5d14d

Dockerfile-Logstash

@@ -0,0 +1,32 @@
+FROM logstash:2.1
+
+RUN apt-get update && apt-get install -y git rubygems python python-pip
+
+# Use kafka-benchmark to produce sample messages
+RUN git clone https://github.com/mre/kafka-benchmark.git /kafka-benchmark \
+    && cd /kafka-benchmark \
+    && echo "26f2fc918f50.load.load.shortterm 0.05 1436357630\n26f2fc918f50.load.load.midterm 0.05 1436357630\n26f2fc918f50.load.load.longterm 0.05 1436357630\n26f2fc918f50.cpu-0.cpu-user 30364 1436357630\n26f2fc918f50.memory.memory-buffered 743657472 1436357630" > message.txt \
+    && pip install -r requirements.txt
+
+# Install the logstash Kafka input plugin
+RUN ./opt/logstash/bin/plugin install logstash-input-kafka
+
+# Install the logstash InfluxDB output plugin
+# Currently InfluxDB 0.9 support is not part of the official logstash influxdb plugin
+# See https://github.com/logstash-plugins/logstash-output-influxdb/issues/24
+# See https://github.com/logstash-plugins/logstash-output-influxdb/pull/29
+# Therefore we add a fork for now which contains fixes for Influxdb 0.9
+# and also allows us to set the InfluxDB measurement name from a field in the graphite string
+RUN mkdir /logstash-plugins \
+    && git clone https://github.com/ultramathman/logstash-output-influxdb.git /logstash-plugins/logstash-output-influxdb \
+    && cd /logstash-plugins/logstash-output-influxdb/ \
+    && gem build logstash-output-influxdb.gemspec \
+    && /opt/logstash/bin/plugin install /logstash-plugins/logstash-output-influxdb/logstash-output-influxdb-2.0.2.gem
+
+ADD contrib/logstash/config.conf /config/logstash-config.conf
+ADD contrib/kafka-benchmark/config.py /kafka-benchmark/config.py
+
+ADD run-logstash.sh run.sh
+RUN chmod +x run.sh
+
+CMD ["./run.sh"]

contrib/kafka-benchmark/config.py

@@ -0,0 +1,7 @@
+config = {
+    'kafka': ['kafka'],
+    'topic': 'metrics',
+    'batches': 1000,
+    'batch_size': 1000,
+    'partitions': [0]
+}

contrib/logstash/README.md

@@ -0,0 +1,3 @@
+This directory contains a sample logstash configuration that use the 
+[logsthash-output-influxdb plugin](https://github.com/logstash-plugins/logstash-output-influxdb).
+The goal is to do a benchmark of kafka-influxdb and logstash.

contrib/logstash/config.conf

@@ -0,0 +1,73 @@
+input
+{
+  kafka
+  {
+    zk_connect => "zk:2181"
+    topic_id => "metrics"
+    auto_offset_reset => "smallest"
+    reset_beginning => true
+    consumer_threads => 1
+    codec => "plain"
+    # Todo: Find out why this is not working :,(
+    #   codec => graphite
+    #	{
+    #		include_metrics => ["[A-Z_]+"]
+    #       metrics => [ "custom.foo", "%{foo}" ]
+    #		fields_are_metrics => true
+    #		metrics_format => "metric_name"
+    #	}
+  }
+}
+filter
+{
+  metrics
+  {
+    meter => "events"
+    add_tag => "metric"
+  }
+  grok
+  {
+    pattern => [ '^(?<metric_name>[^\s]*?) (?<value>[^\s]+?) (?<time>\d+)$']
+  }
+}
+output
+{
+  if "metric" in [tags]
+  {
+    stdout
+    {
+      codec => line
+      {
+        format => "rate: %{[events][rate_1m]}"
+      }
+    }
+  }
+  else
+  {
+    influxdb
+    {
+      host => "influxdb"
+      port => "8086"
+      db => "metrics"
+      retention_policy => "default"
+      user => "root"
+      password => "root"
+      allow_measurement_override => true
+      data_points =>
+      {
+        "measurement" => "%{[metric_name]}"
+        "time" => "%{[time]}"
+        "value" => "%{[value]}"
+      }
+      allow_time_override => true
+      time_precision => "s"
+      coerce_values =>
+      {
+        "time" => "integer"
+      }
+      flush_size => "1000"
+      idle_flush_time => "1"
+     }
+     #	stdout { codec => rubydebug }
+  }
+}

docker-compose-logstash.yml

@@ -0,0 +1,39 @@
+# Creates the following container setup:
+# collectd -> Kafka -> kafka-influxdb -> InfluxDB
+zookeeper:
+  image: wurstmeister/zookeeper
+  ports:
+    - "2181:2181"
+kafka:
+  image: wurstmeister/kafka
+  ports:
+    - "9092:9092"
+  links:
+    - zookeeper:zk
+  volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+  environment:
+    KAFKA_ADVERTISED_HOST_NAME: 192.168.99.101
+    KAFKA_CREATE_TOPICS: "metrics:1:1"
+    KAFKA_ZOOKEEPER_CONNECT: "zk"
+influxdb:
+  image: tutum/influxdb:0.9
+  environment:
+    PRE_CREATE_DB: "metrics"
+  volumes:
+    - ./contrib/influxdb/config.toml:/config/config.toml
+  ports:
+   - "2003"
+   - "8083"
+   - "8086"
+logstash:
+  build: .
+  dockerfile: Dockerfile-Logstash
+  # Uncomment the following three lines for interactive mode:
+  # entrypoint: /bin/bash
+  # stdin_open: true
+  # tty: true
+  links:
+    - zookeeper:zk
+    - kafka:kafka
+    - influxdb:influxdb

docker-compose.yml

@@ -7,13 +7,13 @@ zookeeper:
 kafka:
   image: wurstmeister/kafka
   ports:
-    - "9092"
+    - "9092:9092"
   links:
     - zookeeper:zk
   volumes:
     - /var/run/docker.sock:/var/run/docker.sock
   environment:
-    KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
+    KAFKA_ADVERTISED_HOST_NAME: 192.168.99.101
     KAFKA_CREATE_TOPICS: "metrics:1:1,benchmark:1:1,my_topic:1:1"
     KAFKA_ZOOKEEPER_CONNECT: "zk"
 collectd:

run-logstash.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+KAFKA_HOST="kafka"
+KAFKA_PORT=9092
+INFLUXDB_HOST="influxdb"
+INFLUXDB_PORT=8086
+
+while true
+do
+  echo "Waiting for Kafka connection at $KAFKA_HOST:$KAFKA_PORT. This may take a while..."
+  timeout 1 bash -c "cat < /dev/null > /dev/tcp/$KAFKA_HOST/$KAFKA_PORT"
+  if [ $? -eq 0 ]; then
+    break
+  fi
+  sleep 2
+done
+
+
+while true
+do
+  echo "Waiting for InfluxDB connection at $INFLUXDB_HOST:$INFLUXDB_PORT. This may take a while..."
+  curl --silent -G "http://$INFLUXDB_HOST:$INFLUXDB_PORT/query" --data-urlencode "q=SHOW DATABASES" > /dev/null
+  if [ $? -eq 0 ]; then
+    break
+  fi
+  sleep 2
+done
+
+echo "Starting performance benchmark"
+cd /kafka-benchmark/ && python benchmark.py && logstash --config /config/logstash-config.conf