Fixes #34 - Add Application Performance Monitoring (APM) to the stack

- Upgraded vagrant machines' spec
- Added APM to elastic stack but not tested yet
- Increased number of proxy tasks

README.md

@@ -1,7 +1,7 @@
 [![Build Status on Travis](https://travis-ci.org/shazChaudhry/docker-elastic.svg?branch=server.basePath "CI build status on Travis")](https://travis-ci.org/shazChaudhry/docker-elastic)
 
 # User story
-As a DevOps team member, I want to install [Elastic Stack](https://www.elastic.co/products) so that all application and system logs are collected centrally for searching, visualizing, analyzing and reporting purpose
+As a DevOps team member, I want to install [Elastic Stack](https://www.elastic.co/products) _(v7.7.0 by default)_ so that all application and system logs are collected centrally for searching, visualizing, analyzing and reporting purpose
 
 <p align="center">
   <img src="./pics/elastic-products.PNG" alt="Elastic products" style="width: 400px;"/>
@@ -21,8 +21,11 @@ The architecture used is shown in the table below
     <th>Not in scope</th>
   </tr>
   <tr>
-    <td><img src="./pics/elastic_stack_without_logstash.png" alt="Elastic Stack" style="width: 400px;"/></td>
-    <td>Only beats for log files and metrics are used. 2x Elasticsearch and 1x Kibana are used</td>
+    <td><img src="./pics/elastic-stack-arch.png" alt="Elastic Stack" style="width: 400px;"/></td>
+    <td>
+      Only beats for log files and metrics are used. All logs and metrics are shipped to elasticsearch directly in this repo. </br>
+      2x Elasticsearch, 1x apm-server and 1x Kibana are used.
+    </td>
     <td>Ingest nodes are not used</td>
   </tr>
   <tr>
@@ -35,11 +38,10 @@ The architecture used is shown in the table below
 For the full list of free features that are included in the basic license, see: https://www.elastic.co/subscriptions
 
 # Prerequisite
-* A docker swarm mode cluster allocated to running Elastic Stack must have at least two nodes; 1x master and 1x worker
-* A docekr swarm mode cluster allocated to running containerized custom applications must have at least on node; 1x master
-* On each Elasticsearch cluster node, maximum map count check should be set to as follows:  _(required to run Elasticsearch)_
+* One docker swarm mode cluster allocated to running Elastic Stack. This cluster must have at least two nodes; 1x master and 1x worker. On each Elasticsearch cluster node, maximum map count check should be set to as follows:  _(required to run Elasticsearch)_
   * `sudo sysctl -w vm.max_map_count=262144`
   * `sudo echo 'vm.max_map_count=262144' >> /etc/sysctl.conf` (to persist reboots)
+* One docekr swarm mode cluster allocated to running containerized custom applications. This cluster must have at least on node; 1x master
 
 # Get docker compose files
 You will need these files to deploy Eleasticsearch, Logstash, Kibana, and Beats. So, first SSH in to the master node of the Docker Swarm cluster allocated to running Elastic Stack and clone this repo by following these commands:
@@ -116,6 +118,12 @@ Here is another example:
 * Login to Kibana and you should see traffic coming into Elasticsearch under `logstash-*` index
 * You can use syslog as well as TLS if you wish to add in your own certs
 
+# Testing with APM Java Agent
+Follow these instructions to build a java app that we will use for APM:
+```
+WIP
+```
+
 # References
 - [elastic/examples](https://github.com/elastic/examples) - Home for Elasticsearch examples available to everyone. It's a great way to get started
 - [Elasticsearch sizing and capacity planning](https://www.elastic.co/webinars/elasticsearch-sizing-and-capacity-planning) 

Vagrantfile

@@ -22,7 +22,7 @@ Vagrant.configure("2") do |config|
 		node1.vm.network :private_network, ip: "192.168.99.101"
 		node1.vm.provider :virtualbox do |v|
 			v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-			v.customize ["modifyvm", :id, "--memory", 3000]
+			v.customize ["modifyvm", :id, "--memory", 8000]
 			v.customize ["modifyvm", :id, "--name", "node1"]
 		end
 		node1.vm.provision :shell, inline: $docker_swarm_init
@@ -33,7 +33,7 @@ Vagrant.configure("2") do |config|
 		node2.vm.network :private_network, ip: "192.168.99.102"
 		node2.vm.provider :virtualbox do |v|
 			v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-			v.customize ["modifyvm", :id, "--memory", 3000]
+			v.customize ["modifyvm", :id, "--memory", 8000]
 			v.customize ["modifyvm", :id, "--name", "node2"]
 		end
 		node2.vm.provision :shell, inline: "docker swarm join --token $(cat /vagrant/worker_token) 192.168.99.101:2377"
@@ -44,7 +44,7 @@ Vagrant.configure("2") do |config|
 		node3.vm.network :private_network, ip: "192.168.99.103"
 		node3.vm.provider :virtualbox do |v|
 			v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-			v.customize ["modifyvm", :id, "--memory", 2000]
+			v.customize ["modifyvm", :id, "--memory", 4000]
 			v.customize ["modifyvm", :id, "--name", "node3"]
 		end
 		node3.vm.provision :shell, inline: "docker swarm init --advertise-addr 192.168.99.103 --listen-addr 192.168.99.103:2377"

Vagrantfile.README.md

@@ -1,7 +1,6 @@
 ### Vagrant Box prerequisites
-- The user has admin privileges on the development machine
-- This solution was developed on a Windows 10 pro machine and so a similar machine is aussumed here 
-- At least 9GB of free RAM is available on the machine. Otherwise, Vagrantfile will need editing to adjust the available memory:
+- The user has admin privileges on their development machines
+- At least 20GB of free RAM is available on the machine. Otherwise, Vagrantfile will need editing to adjust the available memory:
   - `v.customize ["modifyvm", :id, "--memory", <MEMORY_ALLOCATION>]`
 - Latest version of [Oracle VM VirtualBox](https://www.virtualbox.org/wiki/Downloads)
 - Latest version of [Git for Windows](https://git-scm.com/downloads)

docker-compose.yml

@@ -26,12 +26,13 @@ services:
       - "80:80"
       - "443:443"
       - "9200:9200"
+      - "8200:8200"
     networks:
       - elastic
     environment:
       - LISTENER_ADDRESS=swarm-listener
       - MODE=swarm
-      - BIND_PORTS=9200
+      - BIND_PORTS=9200,8200
     deploy:
       replicas: 2
 
@@ -62,10 +63,6 @@ services:
         - com.df.servicePath=/
         - com.df.port=9200
         - com.df.srcPort=9200
-    healthcheck:
-      interval: 20s
-      retries: 10
-      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
 
   logstash:
     image: docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-7.7.0}
@@ -101,11 +98,35 @@ services:
         - com.df.servicePath=/
         - com.df.port=5601
         - com.df.srcPort=80
-    healthcheck:
-      interval: 20s
-      retries: 20
-      test: curl -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
-
+
+  apm-server:
+    image: docker.elastic.co/apm/apm-server:${ELASTIC_VERSION:-7.7.0}
+    hostname: "{{.Node.Hostname}}-apm-server"
+    networks:
+      - elastic
+    command: > 
+        --strict.perms=false -e
+        -E apm-server.rum.enabled=true
+        -E setup.kibana.host=kibana:5601
+        -E setup.kibana.username=${ELASTICSEARCH_USERNAME}
+        -E setup.kibana.password=${ELASTICSEARCH_PASSWORD}
+        -E setup.template.settings.index.number_of_replicas=0
+        -E apm-server.kibana.enabled=true
+        -E apm-server.kibana.host=kibana:5601
+        -E apm-server.kibana.username=${ELASTICSEARCH_USERNAME}
+        -E apm-server.kibana.password=${ELASTICSEARCH_PASSWORD}
+        -E output.elasticsearch.hosts=["elasticsearch:9200"]
+        -E output.elasticsearch.username=${ELASTICSEARCH_USERNAME}
+        -E output.elasticsearch.password=${ELASTICSEARCH_PASSWORD}
+        -E xpack.monitoring.enabled=true
+    deploy:
+      labels:
+        - com.df.notify=true
+        - com.df.distribute=true
+        - com.df.servicePath=/
+        - com.df.port=8200
+        - com.df.srcPort=8200
+
 networks:
     elastic:
       external: true