- Adding in Beats integration

Updated the logstash config to allow for both the beats and non-beats log data optionally.

In addition, added in a beats/ directory containing a build of our datadumpbeat along with a
configuration for publishing 10 events/sec.  The period and events_per_period are both tunable
to test various scenarios

beats/.gitattributes

@@ -0,0 +1 @@
+datadumpbeat-*-* binary

beats/.gitignore

@@ -0,0 +1,2 @@
+data/
+logs/

beats/README.md

@@ -0,0 +1,9 @@
+# Overview
+This is a compiled version of a simple beat we created to simulate completely fake, pseudo-random Apache traffic from a Beat.
+
+The short term goals are to simply show how Logstash, Beats, etc. can play nicely together in a testing and lab environment.
+
+# Usage
+To execute the data dump, it's easiest to simply execute from the command line.
+
+`./datadumpbeat-darwin-amd64 -e` - Starts the beat and outputs the logs to the STDERR/STDOUT

beats/datadumpbeat.yml

@@ -0,0 +1,68 @@
+################### datadumpbeat Configuration Example #########################
+
+############################# datadumpbeat ######################################
+
+datadumpbeat:
+  # Defines how often an event is sent to the output
+  period: 1s
+  # Number of events to simulate in that period
+  events_per_period: 10
+  # Locale of the traffic to simulate
+  locale: en
+
+#----------------------------- Logstash output --------------------------------
+output.logstash:
+  # The Logstash hosts
+  hosts: ["localhost:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+logging.selectors: ["*"]
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+#================================ Outputs =====================================
+
+# Configure what outputs to use when sending the data collected by the beat.
+# Multiple outputs may be used.
+
+#-------------------------- Elasticsearch output ------------------------------
+#output.elasticsearch:
+  # Array of hosts to connect to.
+  #hosts: ["localhost:9200"]
+
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+

logstash/logstash.conf

@@ -13,10 +13,14 @@ filter {
     add_field => [ "host_ip", "%{host}" ]
   }
 
-  if [type] == "apache_access" {
+  # Uncomment the remainder of the line to allow the "datadumpbeat" to
+  # be managed in a similar way to normal Apache logs
+
+  if [type] == "apache_access" # or [type] == "datadumpbeat"
+  {
     grok {
       match => {
-        message => "%{IPORHOST:remote_addr} - %{USER:user} \[%{HTTPDATE:timestamp}\] \"%{WORD:http_method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) \"(?:%{URI:referrer}|-)\" %{QS:user_agent}"
+        message => "%{IPORHOST:remote_addr} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:http_method} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) \"(?:%{URI:referrer}|-)\" %{QS:user_agent}"
       }
     }
 
@@ -51,6 +55,13 @@ filter {
   }
 }
 
+# Uncomment for detailed logstash mapping information
+# which can be viewed via `docker-compose logs -f logstash`
+#
+# output {
+#   stdout { codec => rubydebug }
+# }
+
 output {
   elasticsearch {
     hosts => ["elastic1", "elastic2", "elastic3"]