[breaking] Separate logging from default compose. (#94)

* [breaking] Separate logging from default compose.

This commit will introduce a requirement on docker-compose 1.14.0 and a
Boot2Docker version above 17.05 due to bugs in current stable Docker
track.

* Add comment in gitignore

* Share boot2docker image URL between scripts

* Fix README formatting

* Fix scale-workers boot2docker image

.gitignore

@@ -8,3 +8,6 @@ node_modules/
 
 # Ignore generated certificates
 secrets/cert*
+
+# The prod file is generated by the deployment scripts
+docker-compose.prod.yml

README.md

@@ -36,14 +36,9 @@ The use case consists of multiple services, based on Docker images developed in
 
 This use case is primarily about consuming a UI-based analytics website, and we provide only a few of the APIs to the end-user.
 
-* **Analytics UI**\* - `/`, the default UI
-* **Hello Chart** - `/hellochart/`, a temporary UI to test the use-case until we have the Analytics UI in place.
-* **Kibana Dashboard**\*\* - `/kibana/`, use to view logs from the different services.
-* **Swarm Visualizer**\*\*\* - `/viz/`, use to see an overview of the Docker Swarm deployment, and where services are running.
-
-_\* Not available yet._  
-_\*\* Will require authentication and possibly special authorization rights in the future (these are currently accessible by everyone)._  
-_\*\*\* Only available in Swarm mode._
+* **Analytics UI** - `/`, the default UI.
+* **Kibana Dashboard** - `/kibana/`, use to view logs from the different services — only available if the logging stack is included during deployment.
+* **Swarm Visualizer** - `/viz/`, use to see an overview of the deployment, and where services are running. Only available in Swarm mode.
 
 ### Ports
 

docker-compose.logging.yml

@@ -0,0 +1,70 @@
+version: "3.1"
+
+services:
+  elasticsearch:
+    # This service only runs as one instance in our stack and is
+    # used by Kibana (consuming) and Logstash (storing).
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      xpack.security.enabled: "false"
+      xpack.monitoring.enabled: "false"
+      xpack.graph.enabled: "false"
+      xpack.watcher.enabled: "false"
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+
+  logstash:
+    # Logstash runs as one instance on each node in the cluster
+    # since it's using the UDP protocol. Pushes all its collected
+    # logs to the elasticsearch instance.
+    # See https://github.com/qlik-ea/qliktive-custom-analytics-logstash
+    image: qlikea/qliktive-custom-analytics-logstash:master-32
+    ports:
+      - "12201:12201/udp"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+    deploy:
+      mode: global
+
+  kibana:
+    # Used for visualizing and viewing logs stored in elasticsearch.
+    # See https://github.com/qlik-ea/qliktive-kibana-custom-analytics
+    image: qlikea/qliktive-custom-analytics-kibana:master-3
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+
+  openresty:
+    environment:
+      KIBANA_HOST: kibana
+      KIBANA_PORT: 5601
+    logging:
+      driver: "gelf"
+      options:
+        gelf-address: "udp://localhost:12201"
+
+  auth:
+    logging:
+      driver: "gelf"
+      options:
+        gelf-address: "udp://localhost:12201"
+
+  mira:
+    logging:
+      driver: "gelf"
+      options:
+        gelf-address: "udp://localhost:12201"
+
+  qix-engine:
+    logging:
+      driver: "gelf"
+      options:
+        gelf-address: "udp://localhost:12201"
+
+  qix-session:
+    logging:
+      driver: "gelf"
+      options:
+        gelf-address: "udp://localhost:12201"

docker-compose.yml

@@ -16,7 +16,9 @@ services:
     #  - "80:80"
     environment:
       SESSION_COOKIE_NAME: custom-analytics
-      KIBANA_HOST: kibana
+      # kibana is defined in docker-compose.logging.yml, this is just here to
+      # satisfy openresty DNS resolve:
+      KIBANA_HOST: 127.0.0.1
       KIBANA_PORT: 5601
       VISUALIZER_HOST: viz
       VISUALIZER_PORT: 8080
@@ -26,10 +28,6 @@ services:
       AUTH_PORT: 3000
       CERT_FILE: /run/secrets/cert-gateway.crt
       CERT_KEY: /run/secrets/cert-gateway.key
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://localhost:12201"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -59,45 +57,6 @@ services:
       COOKIE_SIGNING_FILE: /run/secrets/cookie_signing
       SUCCESS_REDIRECT_URL: /auth-test
       FAILURE_REDIRECT_URL: /login-fail
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://localhost:12201"
-    deploy:
-      placement:
-        constraints: [node.role == manager]
-
-  elasticsearch:
-    # This service only runs as one instance in our stack and is
-    # used by Kibana (consuming) and Logstash (storing).
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
-    environment:
-      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
-      xpack.security.enabled: "false"
-      xpack.monitoring.enabled: "false"
-      xpack.graph.enabled: "false"
-      xpack.watcher.enabled: "false"
-    deploy:
-      placement:
-        constraints: [node.role == manager]
-
-  logstash:
-    # Logstash runs as one instance on each node in the cluster
-    # since it's using the UDP protocol. Pushes all its collected
-    # logs to the elasticsearch instance.
-    # See https://github.com/qlik-ea/qliktive-custom-analytics-logstash
-    image: qlikea/qliktive-custom-analytics-logstash:master-32
-    ports:
-      - "12201:12201/udp"
-    environment:
-      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
-    deploy:
-      mode: global
-
-  kibana:
-    # Used for visualizing and viewing logs stored in elasticsearch.
-    # See https://github.com/qlik-ea/qliktive-kibana-custom-analytics
-    image: qlikea/qliktive-custom-analytics-kibana:master-3
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -123,10 +82,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       - QIX_ENGINE_IMAGE_NAME=qlikea/engine
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://localhost:12201"
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -140,10 +95,6 @@ services:
     volumes:
       - /home/docker/data/csv:/data
       - /home/docker/data/doc:/doc
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://localhost:12201"
     deploy:
       mode: global
       placement:
@@ -155,10 +106,6 @@ services:
     # to place the session on.
     # See https://github.com/qlik-ea/qliktive-qix-session-service
     image: qlikea/qliktive-qix-session-service:0.0.1-26
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://localhost:12201"
     deploy:
       placement:
         constraints: [node.role == manager]

scripts/boot2docker-iso.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+function boot2docker_iso() {
+  echo "https://github.com/boot2docker/boot2docker/releases/download/v17.06.0-ce-rc3/boot2docker.iso"
+}

scripts/create-swarm-cluster.sh

@@ -6,6 +6,7 @@ set -e
 MANAGERS=1
 WORKERS=2
 USERNAME=$(id -u -n)
+source scripts/boot2docker-iso.sh
 
 print_usage () {
   echo
@@ -44,7 +45,7 @@ if [[ -z "$DEPLOYMENT" ]]; then
   exit 1
 elif [ $DEPLOYMENT == "vsphere" ] || [ $DEPLOYMENT == "VSPHERE" ]; then
   DRIVER=vmwarevsphere
-  SWITCH=
+  SWITCH="--vmwarevsphere-boot2docker-url $(boot2docker_iso)"
 else
   # Windows - Use HyperV and determine HyperV virtual switch.
   if [[ $(uname -o) == "Msys" ]]; then
@@ -61,11 +62,11 @@ else
     fi
 
     DRIVER=hyperv
-    SWITCH="--hyperv-memory 2048 --hyperv-virtual-switch $SWITCH"
+    SWITCH="--hyperv-memory 2048 --hyperv-boot2docker-url $(boot2docker_iso) --hyperv-virtual-switch $SWITCH"
   else
     # Non-Windows - Use Virtualbox and omit virtual switch.
     DRIVER=virtualbox
-    SWITCH="--virtualbox-memory 2048"
+    SWITCH="--virtualbox-memory 2048 --virtualbox-boot2docker-url $(boot2docker_iso)"
   fi
 fi
 

scripts/deploy-stack.sh

@@ -37,12 +37,14 @@ echo "  Deploying stack to docker swarm"
 echo "========================================================================"
 
 eval $(docker-machine env $MACHINE)
-docker stack deploy -c ./docker-compose.yml --with-registry-auth custom-analytics
+docker-compose -f docker-compose.yml -f docker-compose.logging.yml pull
+docker-compose -f docker-compose.yml -f docker-compose.logging.yml config > docker-compose.prod.yml
+docker stack deploy -c ./docker-compose.prod.yml --with-registry-auth custom-analytics
 
 echo "\n$(docker service ls)"
 echo "${BYellow}\nThen all the replicas for the service is started (this may take several minutes) -${Reset}"
 echo
 echo "${BYellow}The following routes can be accessed:${Reset}"
-echo "CUSTOM ANALYTICS         - https://$MACHINEIP/hellochart/"
+echo "CUSTOM ANALYTICS         - https://$MACHINEIP/"
 echo "KIBANA                   - https://$MACHINEIP/kibana/"
 echo "DOCKER SWARM VISUALIZER  - https://$MACHINEIP/viz/"

scripts/scale-workers.sh

@@ -2,6 +2,7 @@
 
 cd "$(dirname "$0")" # change execution directory due to use of relative paths
 USERNAME=$(id -u -n)
+source scripts/boot2docker-iso.sh
 
 print_usage () {
   echo
@@ -47,15 +48,15 @@ EXISTING_MACHINES=$(docker-machine ls)
 
 if [[ $EXISTING_MACHINES == *"vmwarevsphere"* ]]; then
   DRIVER=vmwarevsphere
-  SWITCH=
+  SWITCH="--vmwarevsphere-boot2docker-url $(boot2docker_iso)"
 elif [[ $EXISTING_MACHINES == *"hyperv"* ]]; then
   DRIVER=hyperv
   # Get virtual switch used for HyperV in Windows
   VIRTUAL=$(docker-machine inspect $USERNAME-docker-manager1 | sed -n 's/.*"VSwitch": "\(.*\)",/\1/p')
-  SWITCH="--hyperv-memory 2048 --hyperv-virtual-switch $VIRTUAL"
+  SWITCH="--hyperv-memory 2048 --hyperv-boot2docker-url $(boot2docker_iso) --hyperv-virtual-switch $VIRTUAL"
 elif [[ $EXISTING_MACHINES == *"virtualbox"* ]]; then
   DRIVER=virtualbox
-  SWITCH="--virtualbox-memory 2048"
+  SWITCH="--virtualbox-memory 2048 --virtualbox-boot2docker-url $(boot2docker_iso)"
 else
   echo "No existing deployment found or unknown driver is used"
   print_usage