starting on configs

.gitignore

@@ -1,5 +1,6 @@
 go-audit
 go-audit.yaml
+!examples/**
 *.pprof
 *.test
 *.deb

examples/README.md

@@ -0,0 +1,16 @@
+## examples ##
+
+The following folders contain configs for each program, together they should give you a strong
+starting point for running all of this in production.
+
+These configs are targeted for everything running on a single Ubuntu 14.04 or 16.04 host. They _should_
+work for other distributions but may require some modification.
+
+Set everything up in the following order:
+
+1. [`elasticsearch`](./elasticsearch)
+1. [`streamstash`](./streamstash)
+1. [`rsyslog`](./rsyslog)
+1. [`go-audit`](./go-audit)
+1. [`kibana`](./kibana)
+1. [`elastalert`](./elastalert)

examples/elastalert/README.md

@@ -0,0 +1,57 @@
+## elastalert ##
+
+### Things to install
+
+- `python`
+- `python-dev`
+- `pip`
+- `gcc`
+
+On Ubuntu:
+
+```
+sudo apt install python python-dev python-pip gcc
+```
+
+[elastalert docs](http://elastalert.readthedocs.io/en/latest/running_elastalert.html#downloading-and-configuring) has a
+good guide to getting setup.
+
+A TLDR version:
+
+```
+sudo pip install --upgrade setuptools pip
+cd /opt
+sudo git clone https://github.com/Yelp/elastalert.git
+cd elastalert
+sudo python setup.py install
+sudo pip install -r requirements.txt
+# just answer the defaults for this one
+elastalert-create-index --host localhost --port 9200 --no-ssl --no-auth
+```
+
+Place the files:
+
+- [`elastalert.yaml`](./elastalert.yaml)
+- [`run_uptime.yaml`](./run_uptime.yaml)
+- [`systemd.service`](./systemd.service) - if running `systemd`
+- [`upstart.conf`](./upstart.conf) - if running `upstart`
+
+Logs will be sent to syslog, usually end up at `/var/log/syslog`
+
+Once all that is done you can test the `run_uptime.yaml` rule with (you may want to run `uptime` first)
+
+```
+uptime
+elastalert-test-rule --config /etc/elastalert.yaml /opt/elastalert_rules/run_uptime.yaml
+```
+
+You should see a big json blob of you running `uptime`!
+
+Start or restart `elastalert`
+
+- 14.04 - `sudo start elastalert`
+- 16.04 - `sudo systemctl start elastalert.service`
+
+Logs will be sent to syslog, usually end up at `/var/log/syslog`
+
+Alerts will be sent to `/tmp/alerts`

examples/elastalert/elastalert.yaml

@@ -0,0 +1,48 @@
+# /etc/elastalert.yaml
+rules_folder: /opt/elastalert_rules
+
+# How often ElastAlert will query Elasticsearch
+# The unit can be anything from weeks to seconds
+run_every:
+  seconds: 5
+
+# ElastAlert will buffer results from the most recent
+# period of time, in case some log sources are not in real time
+buffer_time:
+  minutes: 45
+
+# The Elasticsearch hostname for metadata writeback
+# Note that every rule can have its own Elasticsearch host
+es_host: 127.0.0.1
+
+# The Elasticsearch port
+es_port: 9200
+
+# Optional URL prefix for Elasticsearch
+#es_url_prefix: elasticsearch
+
+# Connect with TLS to Elasticsearch
+#use_ssl: True
+
+# Verify TLS certificates
+#verify_certs: True
+
+# GET request with body is the default option for Elasticsearch.
+# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
+# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
+# for details
+#es_send_get_body_as: GET
+
+# Option basic-auth username and password for Elasticsearch
+#es_username: someusername
+#es_password: somepassword
+
+# The index on es_host which is used for metadata storage
+# This can be a unmapped index, but it is recommended that you run
+# elastalert-create-index to set a mapping
+writeback_index: elastalert_status
+
+# If an alert fails for some reason, ElastAlert will retry
+# sending the alert until this time period has elapsed
+alert_time_limit:
+  days: 2

examples/elastalert/run_uptime.yaml

@@ -0,0 +1,18 @@
+# /opt/elastalert_rules/run_uptime.yaml
+name: go-audit run uptime
+index: streamstash-%Y.%m.%d
+use_strftime_index: true
+type: any
+filter:
+- query:
+    query_string:
+      query: go-audit.execve.command:uptime
+# write alerts to /tmp/alerts for debugging purposes
+alert:
+- command
+pipe_match_json: true
+command: ["/usr/bin/tee", "-a", "/tmp/alerts"]
+# Enable email alerts:
+#alert:
+#- email
+#email: YOUR-EMAIL@SOMEWHERE.COM

examples/elastalert/systemd.service

@@ -0,0 +1,11 @@
+# /etc/systemd/system/elastalert.service
+[Unit]
+Description = elastalert
+After=network.target
+
+[Service]
+Type = simple
+ExecStart = /usr/local/bin/elastalert --config /etc/elastalert.yaml
+
+[Install]
+WantedBy = multi-user.target

examples/elastalert/upstart.conf

@@ -0,0 +1,24 @@
+# /etc/init/elastalert.conf
+description "elastalert"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 10 5
+
+setuid nobody
+setgid nogroup
+
+chdir /opt/elastalert
+
+script
+    set -e
+    rm -f "/tmp/elastalert.log"
+    mkfifo "/tmp/elastalert.log"
+    (setsid logger -t"elastalert" <"/tmp/elastalert.log" &)
+    exec >"/tmp/elastalert.log" 2>"/tmp/elastalert.log"
+    rm "/tmp/elastalert.log"
+
+    exec /usr/local/bin/elastalert --config /etc/elastalert.yaml
+end script

examples/elasticsearch/README.md

@@ -0,0 +1,45 @@
+## elasticsearch ##
+
+Very bare bones approach to getting elasticsearch running
+
+## Things to install ##
+
+- `java`
+- [`elasticsearch`](https://www.elastic.co/downloads/past-releases/elasticsearch-2-4-1) - Avoid using 5.x until [elastalert supports it](https://github.com/Yelp/elastalert/issues/510)
+- [`kopf`](https://github.com/lmenezes/elasticsearch-kopf) - makes ops a lot easier
+
+On Ubuntu 16.04:
+
+```
+sudo apt install openjdk-8-jre-headless
+```
+
+On Ubuntu 14.04:
+
+```
+sudo apt install openjdk-7-jre-headless
+```
+
+On Ubuntu:
+
+```
+wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.1/elasticsearch-2.4.1.deb
+sudo dpkg -i elasticsearch-2.4.1.deb
+```
+
+Place the files
+
+- [`elasticsearch.yml`](./elasticsearch.yml)
+
+Start or restart `elasticsearch`
+
+- 14.04 - `sudo /etc/init.d/elasticsearch start`
+- 16.04 - `sudo systemctl start elasticsearch.service`
+
+Once the service is running apply the [`mapping.json`](./mapping.json) template to prepare for `streamstash` logs
+
+```
+curl -d @mapping.json http://localhost:9200/_template/streamstash
+```
+
+Logs are usually at `/var/log/elasticsearch/elasticsearch.log`

examples/elasticsearch/elasticsearch.yml

@@ -0,0 +1,3 @@
+# /etc/elasticsearch/elasticsearch.yml
+network.host: [ _site_, _local_ ]
+node.name: ${HOSTNAME}

examples/elasticsearch/mapping.json

@@ -0,0 +1,41 @@
+{
+    "template": "streamstash-*",
+    "mappings": {
+        "_default_": {
+            "dynamic_templates": [
+                {
+                    "message_field": {
+                        "mapping": {
+                            "index": "analyzed",
+                            "omit_norms": true,
+                            "type": "string"
+                        },
+                        "match_mapping_type": "string",
+                        "match": "message"
+                    }
+                },
+                {
+                    "string_fields": {
+                        "mapping": {
+                            "index": "analyzed",
+                            "omit_norms": true,
+                            "type": "string",
+                            "fields": {
+                                "raw": {
+                                    "index": "not_analyzed",
+                                    "type": "string"
+                                }
+                            }
+                        },
+                        "match_mapping_type": "string",
+                        "match": "*"
+                    }
+                }
+            ],
+            "_all": {
+                "omit_norms": true,
+                "enabled": true
+            }
+        }
+    }
+}

examples/go-audit/README.md

@@ -0,0 +1,37 @@
+## go-audit ##
+
+The files here will get `go-audit` logging to `rsyslog` and has a decent default ruleset.
+
+An upstart config and systemd unit are provided as well
+
+### Things to install
+
+- `auditd` - the one that comes with your distro is fine, we just need `auditctl` for now
+  - ie: `sudo apt install auditd`
+- [`golang`](https://golang.org/dl/) - so you can compile `go-audit`
+
+On Ubuntu:
+
+```
+sudo apt install auditd golang
+```
+
+To install `go-audit`
+
+```
+make
+sudo cp go-audit /usr/local/bin
+```
+
+Place the files:
+
+- [`go-audit.yaml`](./go-audit.yaml)
+- [`systemd.service`](./systemd.service) - if running `systemd`
+- [`upstart.conf`](./upstart.conf) - if running `upstart`
+
+Start or restart `go-audit`
+
+- 14.04 - `sudo start go-audit`
+- 16.04 - `sudo systemctl start go-audit.service`
+
+Logs will be in `elasticsearch`

examples/go-audit/go-audit.yaml

@@ -0,0 +1,43 @@
+# /etc/go-audit.yaml
+
+canary: true
+
+# use /var/run/go-audit.sock to write events
+output:
+  syslog:
+    attempts: 15
+    enabled: true
+    network: unix
+    address: /var/run/go-audit.sock
+    priority: 132
+    tag: go-audit
+
+# log an event when we believe a message has been lost
+message_tracking:
+  enabled: true
+  log_out_of_order: false
+  max_out_of_order: 500
+
+rules:
+  - -b 1024
+  # required if you set canary: true
+  - -w /proc/net/netlink -p war -k netlink-file
+  # watch interesting network events
+  - -a exit,always -S connect
+  - -a exit,always -S listen
+  # watch execve for everything that has an auid set (ignores things like cron)
+  - -a exit,always -F arch=b64 -F auid!=-1 -S execve -k user_commands
+  - -a exit,always -F arch=b32 -F auid!=-1 -S execve -k user_commands
+  # failure to access file because of perms
+  - -a always,exit -F arch=b32 -S open -S openat -F exit=-EACCES -k access
+  - -a always,exit -F arch=b64 -S open -S openat -F exit=-EACCES -k access
+  - -a always,exit -F arch=b32 -S open -S openat -F exit=-EPERM -k access
+  - -a always,exit -F arch=b64 -S open -S openat -F exit=-EPERM -k access
+
+filters:
+  # reduce the number of connect syscall events being logged
+  - syscall: 42
+    message_type: 1306
+    # 0200....7F - ipv4 on any port to 127.x.x.x
+    # 01 - local/unix domain sockets
+    regex: saddr=(0200....7F|01)

examples/go-audit/systemd.service

@@ -0,0 +1,12 @@
+# /etc/systemd/system/go-audit.service
+[Unit]
+Description = go-audit
+After=network.target auditd.service
+Conflicts = auditd.service
+
+[Service]
+Type = simple
+ExecStart = /usr/local/bin/go-audit -config /etc/go-audit.yaml
+
+[Install]
+WantedBy = multi-user.target

examples/go-audit/upstart.conf

@@ -0,0 +1,24 @@
+# /etc/init/go-audit.conf
+
+description "go-audit"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 10 5
+
+script
+    # redirect stdout and stderr to syslog
+    set -e
+    rm -f "/tmp/go-audit.log"
+    mkfifo "/tmp/go-audit.log"
+    (setsid logger -t"go-audit" <"/tmp/go-audit.log" &)
+    exec >"/tmp/go-audit.log" 2>"/tmp/go-audit.log"
+    rm "/tmp/go-audit.log"
+
+    # There can be only one auditd
+    /etc/init.d/auditd stop || true
+
+    exec /usr/local/bin/go-audit -config=/etc/go-audit.yaml
+end script