Add logstash5 and java lumber + dashboard

urso · May 25, 2016 · 52e4954 · 52e4954
1 parent 0a78757
commit 52e4954
Show file tree

Hide file tree

Showing 62 changed files with 1,777 additions and 11 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,26 @@
+# Directories
+/.vagrant
+/.idea
+/build
+/*/data
+
+# Files
+.DS_Store
+/glide.lock
+/beats.iml
+*.dev.yml
+
+# Editor swap files
+*.swp
+*.swo
+*.swn
+.\#*
+
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+*.exe
+*.test
+*.prof
+*.pyc
diff --git a/dashboard/dashboard.json b/dashboard/dashboard.json
@@ -0,0 +1,214 @@
+[
+  {
+    "_id": "Dashboard",
+    "_type": "dashboard",
+    "_source": {
+      "title": "Dashboard",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":1,\"id\":\"Beats-Throughput\",\"panelIndex\":1,\"row\":1,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-usage\",\"panelIndex\":2,\"row\":5,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"Mem-RSS-Generatorbeat\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":6,\"size_y\":2,\"col\":1,\"row\":9},{\"id\":\"Mem-RSS-Logstash\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":6,\"size_y\":2,\"col\":7,\"row\":9}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{\"P-1\":{\"vis\":{\"colors\":{\"genbeat_logstash_v2_compressed\":\"#7EB26D\",\"genbeat_logstash_v2_compressed_pipelined\":\"#447EBC\",\"genbeat_tstlj_es_compressed\":\"#F9934E\",\"genbeat_tstlj_v2_compressed\":\"#E24D42\",\"genbeat_tstlj_v2_compressed_pipelined\":\"#3F6833\"}}},\"P-2\":{\"vis\":{\"colors\":{\"genbeat_logstash_v2_compressed\":\"#7EB26D\",\"genbeat_logstash_v2_compressed_pipelined\":\"#447EBC\",\"genbeat_tstlj_es_compressed\":\"#F9934E\",\"genbeat_tstlj_v2_compressed\":\"#E24D42\",\"genbeat_tstlj_v2_compressed_pipelined\":\"#3F6833\"}}},\"P-4\":{\"vis\":{\"colors\":{\"genbeat_logstash_v2_compressed_pipelined\":\"#447EBC\",\"genbeat_logstash_v2_compressed\":\"#7EB26D\"}}},\"P-3\":{\"vis\":{\"colors\":{\"genbeat_logstash_v2_compressed_pipelined\":\"#447EBC\",\"genbeat_tstlj_v2_compressed\":\"#E24D42\",\"genbeat_tstlj_v2_compressed_pipelined\":\"#3F6833\",\"genbeat_tstlj_es_compressed\":\"#F9934E\",\"genbeat_logstash_v2_compressed\":\"#7EB26D\"}}}}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-System-Process-(Generatorbeat- -Logstash)",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset System Process (Generatorbeat + Logstash)",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:system AND metricset.name:process AND system.process.cmdline:(generatorbeat logstash)\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-System-Process-Generatorbeat- -Logstash",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset System Process - Generatorbeat + Logstash",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:system AND metricset.name:process AND system.process.cmdline:(generatorbeat logstash)\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-Beats-Throughput",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset Beats Throughput",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:beats AND metricset.name:generatorbeat\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-System-Process",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset System Process",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:system AND metricset.name:process\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-System-Process-Generatorbeat",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset System Process - Generatorbeat",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:system AND metricset.name:process AND system.process.name:generatorbeat\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Metricset-System-Process-Logstash",
+    "_type": "search",
+    "_source": {
+      "title": "Metricset System Process - Logstash",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"collectbeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"metricset.module:system AND metricset.name:process AND system.process.cmdline:(logstash)\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "Beats-Throughput",
+    "_type": "visualization",
+    "_source": {
+      "title": "Beats Throughput",
+      "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":true,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"beats.generatorbeat.total.success\",\"customLabel\":\"events/s\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.run\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"time\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"genbeat_logstash_v2_compressed\":\"#7EB26D\",\"genbeat_logstash_v2_compressed_pipelined\":\"#447EBC\",\"genbeat_tstlj_es_compressed\":\"#F9934E\",\"genbeat_tstlj_v2_compressed\":\"#E24D42\",\"genbeat_tstlj_v2_compressed_pipelined\":\"#BA43A9\"}}}",
+      "description": "",
+      "savedSearchId": "Metricset-Beats-Throughput",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "CPU-usage",
+    "_type": "visualization",
+    "_source": {
+      "title": "CPU usage",
+      "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.cpu.total_p\",\"customLabel\":\"CPU - total\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.run\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"system.process.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "Metricset-System-Process-Generatorbeat- -Logstash",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "Mem-RSS",
+    "_type": "visualization",
+    "_source": {
+      "title": "Mem RSS",
+      "visState": "{\"title\":\"CPU usage\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.mem.rss\",\"customLabel\":\"Mem RSS\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"system.process.name\",\"size\":5,\"order\":\"asc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.run\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "Metricset-System-Process-Generatorbeat- -Logstash",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "Mem-RSS-Generatorbeat",
+    "_type": "visualization",
+    "_source": {
+      "title": "Mem RSS - Generatorbeat",
+      "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.mem.rss\",\"customLabel\":\"Mem RSS\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.run\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "Metricset-System-Process-Generatorbeat",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "Mem-RSS-Logstash",
+    "_type": "visualization",
+    "_source": {
+      "title": "Mem RSS - Logstash",
+      "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.mem.rss\",\"customLabel\":\"Mem RSS\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.run\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "savedSearchId": "Metricset-System-Process-Logstash",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[]}"
+      }
+    }
+  }
+]
diff --git a/env.sh b/env.sh
@@ -1,5 +1,5 @@
 # make sure all child processes are killed
-trap "pkill -P $$" SIGINT SIGTERM EXIT
+trap "pkill -9 -P $$" SIGINT SIGTERM EXIT
 
 # get testing home directory from finding absolute path of env.sh
 DIR=$(dirname $0)
@@ -9,17 +9,25 @@ TEST_HOME=$(cd "$(dirname $BASH_SOURCE)"; pwd)
 : ${DURATION:=60}
 : ${ENABLE_WATCH:=true}
 
+: ${WORKDIR:=$GOPATH/src}
+
 # test executables
-: ${GENERATORBEAT:=$GOPATH/src/github.com/urso/generatorbeat/generatorbeat}
+: ${GENERATORBEAT:=$WORKDIR/github.com/urso/generatorbeat/generatorbeat}
+
+: ${JAVA_LUMBER:=$WORKDIR/github.com/ph/java-lumber}
 
-: ${TST_LJ:=$GOPATH/src/github.com/urso/go-lumber/tst-lj}
+: ${TST_LJ:=$WORKDIR/github.com/urso/go-lumber/tst-lj}
 
-: ${COLLECTBEAT:=$GOPATH/src/github.com/urso/collectbeat/collectbeat}
+: ${COLLECTBEAT:=$WORKDIR/github.com/urso/collectbeat/collectbeat}
 
 : ${EXPVAR_RATES:=$TEST_HOME/scripts/expvar_rates.py}
 
 : ${LOGSTASH:=logstash}
 
+: ${LOGSTASH2=$WORKDIR/logstash-2.3.2/bin/logstash}
+
+: ${LOGSTASH5=$WORKDIR/logstash-5.0.0-alpha2/bin/logstash}
+
 : ${OUTDIR:=$DIR}
 
 : ${OUTCONFIG:=$TEST_HOME/collect_es.yml}
@@ -28,6 +36,12 @@ TEST_HOME=$(cd "$(dirname $BASH_SOURCE)"; pwd)
 
 # run script utilities
 
+gradle_run() {
+    cd $1
+    gradle run &
+    cd -
+}
+
 collect() {
     sleep $WAIT_COLLECT
     if [ -f "$OUTCONFIG" ]; then

diff --git a/genbeat_jlumber_v2/compressed/collect.yml b/genbeat_jlumber_v2/compressed/collect.yml
@@ -0,0 +1,23 @@
+metricbeat.modules:
+- module: system
+  metricsets: ["cpu", "memory", "process"]
+  enabled: true
+  period: 1s
+  processes:
+    - generatorbeat
+    - java
+
+- module: beats
+  metricsets:
+    - generatorbeat
+  enabled: true
+  period: 1s
+
+fields:
+  run: genbeat_jlumber_v2_compressed
+  producer: generatorbeat
+  consumer: java-lumber
+  protocol: lj_v2
+  compression: 3
+  pipelining: 0
+  bulksize: 2048
diff --git a/genbeat_jlumber_v2/compressed/genbeat.yml b/genbeat_jlumber_v2/compressed/genbeat.yml
@@ -0,0 +1,6 @@
+generatorbeat.generators.filebeat:
+  worker: 1
+  repeat: 1
+
+output.logstash:
+  hosts: ["localhost:5044"]
diff --git a/genbeat_jlumber_v2/compressed/run.sh b/genbeat_jlumber_v2/compressed/run.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source $(dirname $0)/../../env.sh
+
+# start lumberjack test server
+gradle_run $JAVA_LUMBER >/dev/null 2>&1
+
+# start generatorbeat sending events to test server
+$GENERATORBEAT -httpprof localhost:6060 -c $DIR/genbeat.yml &
+
+watch http://localhost:6060
+
+collect
+
+wait_shut $1
diff --git a/genbeat_jlumber_v2/compressed_pipelined/collect.yml b/genbeat_jlumber_v2/compressed_pipelined/collect.yml
@@ -0,0 +1,23 @@
+metricbeat.modules:
+- module: system
+  metricsets: ["cpu", "memory", "process"]
+  enabled: true
+  period: 1s
+  processes:
+    - generatorbeat
+    - java
+
+- module: beats
+  metricsets:
+    - generatorbeat
+  enabled: true
+  period: 1s
+
+fields:
+  run: genbeat_jlumber_v2_compressed_pipelined
+  producer: generatorbeat
+  consumer: java-lumber
+  protocol: lj_v2
+  compression: 3
+  pipelining: 5
+  bulksize: 2048
diff --git a/genbeat_jlumber_v2/compressed_pipelined/genbeat.yml b/genbeat_jlumber_v2/compressed_pipelined/genbeat.yml
@@ -0,0 +1,7 @@
+generatorbeat.generators.filebeat:
+  worker: 1
+  repeat: 1
+
+output.logstash:
+  hosts: ["localhost:5044"]
+  pipelining: 5
diff --git a/genbeat_jlumber_v2/compressed_pipelined/run.sh b/genbeat_jlumber_v2/compressed_pipelined/run.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source $(dirname $0)/../../env.sh
+
+# start lumberjack test server
+gradle_run $JAVA_LUMBER >/dev/null 2>&1
+
+# start generatorbeat sending events to test server
+$GENERATORBEAT -httpprof localhost:6060 -c $DIR/genbeat.yml &
+
+watch http://localhost:6060
+
+collect
+
+wait_shut $1