more inital updates

README.md

@@ -1,8 +1,63 @@
-# dogvscat
+# dogvscat (Work In Progress)
 
-A Docker Swarm cluster needs more then just your app running.
+This repo gives a few examples of patterns for how you might build Docker Swarm clusters with all the bells and whistles. 
 
-Sometimes live on [dogvs.cat](http://dogvs.cat)
+A Docker Swarm cluster needs more then just your app running, it often needs at least these additional services:
 
-COMING SOON (May/June 2018)
+- Layer 7 Reverse Proxy (to host multiple HTTP sites on one port)
+- Swarm-aware storage for data persistance
+- Centralized logging of your app containers
+- Centralized monitoring of nodes and containers
+- Cluster management GUI
+- Continuous deployment of updated images
 
+This demo is meant for you to `git clone` and run locally to help you learn the tools and methods for building a complete Docker Swarm cluster.
+
+## Getting Started
+
+This repo holds two deployment examples for Docker Swarm
+
+- Docker Swarm CE (Community Edition) open source stack
+- Docker Swarm EE (Enterprise Edition) stack
+
+The EE stack requires at least a trial license to deploy.
+
+## Deploying the Swarm CE Example
+
+You can do all this locally on a single node or optionally using Docker Machine to multi-node clusters.
+
+### Step 1: Set needed environment variables
+
+The scripts and compose/stack files use variables to make this demo easier to get started. Set these at your shell before running commands
+
+```
+TODO: add envs
+```
+
+### Step 2: (single node local Swarm)
+
+Just have Docker installed, either via Docker for Windows/Mac or on Linux. See my [YouTube videos on the proper way to setup your OS for Docker](https://www.youtube.com/watch?v=Fc7Rjll30jY&list=PL6cactdCCnTLqhFgmXAVdwLPCM_SZdGYq) using downloads from [store.docker.com](https://store.docker.com).
+
+Then just create a single-node Swarm in that engine:
+
+`docker swarm init`
+
+### Step 2: (multi-node docker-machine Swarm)
+
+`./create-servers.sh` gives example docker-machine commands for creating 3 nodes in various VM environments including locally with VirtualBox, Hyper-V, and in the cloud using Digital Ocean.
+
+### Step 3: Enable Docker Engine Metrics
+
+### Step 4: Initialize Swarm and Join Nodes
+
+### Step 5: Enable Persistent Storage with REX-Ray
+
+### Step 6: Deploy Reverse Proxy using Traefik
+
+### Step 7: Deploy Ops Tools: ELK, Prometheus, and Portainer
+
+### Step 8: Deploy sample apps and test
+
+## Deploying the Swarm EE Example
+
+TODO

create-servers.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -x
+# set -x
 
 # create managers servers in digital ocean with pre-set environment vars
 # https://docs.docker.com/machine/drivers/digital-ocean/

delete-servers.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
+set -x
 
-# create managers servers
+# delete all docker machines starting with dvc
 for server in {1..3}; do
 docker-machine rm -y dvc${server} &
 done
 
-# create servers
-
+# delete all storage in DO (be sure you are ok deleting ALL storage in an account)
+doctl compute volume ls --format ID --no-header | while read -r id; do doctl compute volume rm -f "$id"; done
 
 
 

generate-some-votes.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# create POST data files with ab friendly formats
+python make-data.py
+
+# create 3000 votes
+ab -n 1000 -c 50 -p posta -T "application/x-www-form-urlencoded" http://vote.dogvs.cat/
+ab -n 1000 -c 50 -p postb -T "application/x-www-form-urlencoded" http://vote.dogvs.cat/
+ab -n 1000 -c 50 -p posta -T "application/x-www-form-urlencoded" http://vote.dogvs.cat/

hash-config-secret.sh

@@ -1,3 +1,5 @@
 #!/bin/sh
+set -x
+
 export LOGSTASH_CONF=$(shasum logstash.conf -a 512 | cut -c1-16)
 

make-data.py

@@ -0,0 +1,13 @@
+# this creates urlencode-friendly files without EOL
+import urllib
+outfile = open('postb', 'w')
+params = ({ 'vote': 'b' })
+encoded = urllib.urlencode(params)
+outfile.write(encoded)
+outfile.close()
+outfile = open('posta', 'w')
+params = ({ 'vote': 'a' })
+encoded = urllib.urlencode(params)
+outfile.write(encoded)
+outfile.close()
+

menu/index.html

@@ -21,7 +21,6 @@ <h1 class="display-1">Welcome to Dog vs. Cat</h1>
             <li><h1><a href="http://vote.dogvs.cat/">Dog vs. Cat GO VOTE!</a></h1></li>
             <li><h1><a href="http://result.dogvs.cat/">Dog vs. Cat WATCH VOTE RESULTS</a></h1></li>
             <li><h1><a href="http://ghost.dogvs.cat/">Ghost (blog with MySQL backend)</a></h1></li>
-            <li><h1><a href="http://viz.dogvs.cat/">Swarm Visualizer</a></h1></li>
             <li><h1><a href="http://portainer.dogvs.cat/">Portainer (Swarm GUI management)</a></h1></li>
             <li><h1><a href="http://kibana.dogvs.cat/">Kibana (ELK log search)</a></h1></li>
             <li><h1><a href="http://prom.dogvs.cat/">Prometheus (monitoring console)</a></h1></li>
@@ -30,6 +29,7 @@ <h1 class="display-1">Welcome to Dog vs. Cat</h1>
             <li><h3>Unsee (monitoring alert dashbaord) (coming)</h3></li>
             <li><h3>OpenFaaS (self-hosted functions) (coming)</h3></li>
             <li><h3>Portus (self-hosted registry GUI) (coming)</h3></li>
+            <li><h3>GitLab CI/CD (coming)</h3></li>
             <li><h3>Drone CI/CD (coming)</h3></li>
           </ul>
         </div>

posta

@@ -0,0 +1 @@
+vote=a

postb

@@ -0,0 +1 @@
+vote=b

stack-ee-ghost.yml

@@ -1,5 +1,72 @@
-version: '3.5'
+version: '3.2'
+
+services:
+
+  ghost:
+    image: ghost:1-alpine
+    networks:
+      - proxy
+      - ghost
+    environment:
+      # see https://docs.ghost.org/docs/config#section-running-ghost-with-config-env-variables
+      database__client: mysql
+      database__connection__host: db
+      database__connection__user: root
+      database__connection__password: OpdP2dy4jzAT
+      database__connection__database: ghost
+      url: http://ghost.dogvs.cat
+    deploy:
+      replicas: 1
+      labels:
+        - traefik.port=2368
+        - traefik.docker.network=proxy
+        - traefik.frontend.rule=Host:ghost.dogvs.cat
+  db:
+    image: mysql:5.7
+    volumes:
+      - db:/var/lib/mysql
+    secrets:
+      - ghost-db-password
+    networks:
+      - ghost
+    deploy:
+      endpoint_mode: dnsrr
+    environment:
+      MYSQL_ROOT_PASSWORD_FILE: /run/secrets/ghost-db-password
+
+  db-backup:
+    image: mysql:5.7
+    command: sh -c "while true; do /usr/bin/mysqldump -u root --password=$$(< $$MYSQL_ROOT_PASSWORD_FILE) --all-databases --host=db > /backup/backup$$(date +'%H').sql ; sleep 3600; done"
+    volumes:
+      - db-backup:/backup
+    secrets:
+      - ghost-db-password
+    networks:
+      - ghost
+    environment:
+      MYSQL_ROOT_PASSWORD_FILE: /run/secrets/ghost-db-password
+
+  #TODO: backup for static content
+
+
+networks:
+  ghost: {}
+  proxy:
+    external: true
 
 volumes:
   db:
-    driver: cloudstor:aws 
+    driver: cloudstor:latest
+    driver_opts:
+      size: 1
+  db-backup:
+    driver: cloudstor:latest
+    driver_opts:
+      size: 1
+
+secrets:
+  ghost-db-password:
+    external: true
+
+
+

stack-ghost.yml

@@ -21,10 +21,10 @@ services:
         - traefik.port=2368
         - traefik.docker.network=proxy
         - traefik.frontend.rule=Host:ghost.dogvs.cat
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
   db:
     image: mysql:5.7
@@ -38,11 +38,34 @@ services:
       endpoint_mode: dnsrr
     environment:
       MYSQL_ROOT_PASSWORD_FILE: /run/secrets/db-password
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
-
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
+
+
+  db-backup:
+    image: mysql:5.7
+    command: sh -c "while true; do /usr/bin/mysqldump -u root --password=$$(< $$MYSQL_ROOT_PASSWORD_FILE) --all-databases --host=db > /backup/backup$$(date +'%H').sql ; sleep 3600; done"
+    volumes:
+      - db-backup:/backup
+    secrets:
+      - db-password
+    networks:
+      - ghost
+    environment:
+      MYSQL_ROOT_PASSWORD_FILE: /run/secrets/db-password
+    deploy:
+      restart_policy:
+        delay: 600s
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
+
+  #TODO: backup for static content
+
+
 networks:
   ghost: {}
   proxy:
@@ -53,6 +76,10 @@ volumes:
     driver: rexray/dobs
     driver_opts:
       size: 1
+  db-backup:
+    driver: rexray/dobs
+    driver_opts:
+      size: 1
 
 secrets:
   db-password:

stack-menu.yml

@@ -12,10 +12,10 @@ services:
         - traefik.port=80
         - traefik.docker.network=proxy
         - traefik.frontend.rule=Host:www.dogvs.cat,dogvs.cat
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
     networks:
       - proxy
 

stack-prune.yml

@@ -8,7 +8,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     deploy:
       mode: global
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "tcp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "tcp://127.0.0.1:5000"

stack-voting.yml

@@ -11,10 +11,10 @@ services:
       - frontend
     deploy:
       endpoint_mode: dnsrr
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
   db:
     image: postgres:9.6
@@ -24,10 +24,10 @@ services:
       - backend
     deploy:
       endpoint_mode: dnsrr
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
   vote:
     image: bretfisher/examplevotingapp_vote
@@ -40,10 +40,10 @@ services:
         - traefik.port=80
         - traefik.docker.network=proxy
         - traefik.frontend.rule=Host:vote.dogvs.cat
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
   result:
     image: bretfisher/examplevotingapp_result
@@ -55,10 +55,10 @@ services:
         - traefik.port=80
         - traefik.docker.network=proxy
         - traefik.frontend.rule=Host:result.dogvs.cat
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
   worker:
     image: bretfisher/examplevotingapp_worker:java
@@ -67,10 +67,10 @@ services:
       - backend
     deploy:
       replicas: 1
-    logging:
-      driver: "gelf"
-      options:
-        gelf-address: "udp://127.0.0.1:5000"
+    # logging:
+    #   driver: "gelf"
+    #   options:
+    #     gelf-address: "udp://127.0.0.1:5000"
 
 networks:
   frontend: {}